diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 3a24008..80fe0de 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -110,7 +110,7 @@
            
         main.LogFileName = main.logdir + "/" + main.TEST + "_" +str(currentTime) + ".log"
         main.ReportFileName = main.logdir + "/" + main.TEST + "_" + str(currentTime) + ".rpt"
-        main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"               
+        main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
  
         #### Add log-level - Report
         logging.addLevelName(9, "REPORT")
@@ -229,14 +229,14 @@
         #main.log.report(testResult)
         main.testResult = testResult
         main.log.exact(testResult)
-        
+
         ##CSV output needed for Jenkin's plot plugin
         #NOTE: the elements were orded based on the colors assigned to the data
         logfile = open(main.JenkinsCSV ,"w")
         logfile.write(",".join( ['Tests Failed', 'Tests Passed', 'Tests Planned'] ) + "\n")
-        logfile.write(",".join( [str(main.TOTAL_TC_FAIL), str(main.TOTAL_TC_PASS), str(main.TOTAL_TC_PLANNED)] ))
-        logfile.close()        
-        
+        logfile.write(",".join( [str(int(main.TOTAL_TC_FAIL)), str(int(main.TOTAL_TC_PASS)), str(int(main.TOTAL_TC_PLANNED))] ))
+        logfile.close()
+
 
 
 
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
index 235d138..0f879b6 100755
--- a/TestON/dependencies/rotate.sh
+++ b/TestON/dependencies/rotate.sh
@@ -51,4 +51,4 @@
 #Maybe this should be an argument? pack-and-rotate supports that
 nr_max=20
 
-pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap" ${nr_max}
+pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap ${root_dir}/${base_name}*.log" ${nr_max}
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 17c25a0..bf3a2c6 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -1207,7 +1207,7 @@
                         break
             mn_ports.sort(key=float)
             onos_ports.sort(key=float)
-            #print "\nPorts for Switch %s:" % (switch['name'])
+            #print "\nPorts for Switch %s:" % (mn_switch['name'])
             #print "\tmn_ports[] = ", mn_ports
             #print "\tonos_ports[] = ", onos_ports
             mn_ports_log = mn_ports
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 1795a16..ae0f599 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -114,7 +114,7 @@
         args = utilities.parse_args(["SRC","TARGET","PINGTIME"],**pingParams)
         precmd = "sudo rm /tmp/ping." + args["SRC"]
         self.execute(cmd=precmd,prompt="(.*)",timeout=10)
-        command = "sudo mininet/util/m " + args["SRC"] + " ping "+args ["TARGET"]+" -i .2 -w " + str(args['PINGTIME']) + " > /tmp/ping." + args["SRC"] + " &"
+        command = "sudo mininet/util/m " + args["SRC"] + " ping "+args ["TARGET"]+" -i .2 -w " + str(args['PINGTIME']) + " -D > /tmp/ping." + args["SRC"] + " &"
         main.log.info( command ) 
         self.execute(cmd=command,prompt="(.*)",timeout=10)
         self.handle.sendline("")
@@ -372,57 +372,40 @@
 
     def get_flowTable(self, protoVersion, sw):
         #TODO document usage
-        #FIXME: clean up print statements and such
+        #TODO add option to look at cookies. ignoreing them for now
         self.handle.sendline("cd")
         self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
-        #TODO: Write seperate versions of the function for this, possibly a string that tells it which switch is in use?
-        #For 1.0 version of OVS
-        #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $6 $7 }' |sort -n -k1"
-        #for 1.3 version of OVS
-        #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $3 $7 $8}' |sort -n -k1"
+        #print "get_flowTable(" + str(protoVersion) +" " + str(sw) +")"
         #NOTE: Use format to force consistent flow table output across versions
         if protoVersion==1.0:
-            command = "sudo ovs-ofctl dump-flows " + sw + " -F OpenFlow10-table_id | awk '{OFS=\",\" ; print $1  $3  $6  $7  $8}' |sort -n -k1"
+            command = "sudo ovs-ofctl dump-flows " + sw + " -F OpenFlow10-table_id | awk '{OFS=\",\" ; print $1  $3  $6  $7  $8}' | cut -d ',' -f 2- | sort -n -k1 -r"
             self.handle.sendline(command)
-            self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
+            self.handle.expect(["k1 -r",pexpect.EOF,pexpect.TIMEOUT])
             self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
             response = self.handle.before
             #print "response=", response
             return response
         elif protoVersion==1.3:
-            command = "sudo ovs-ofctl dump-flows " + sw + " -O OpenFlow13  | awk '{OFS=\",\" ; print $1  $3  $6  $7}' |sort -n -k1" 
+            command = "sudo ovs-ofctl dump-flows " + sw + " -O OpenFlow13  | awk '{OFS=\",\" ; print $1  $3  $6  $7}' | cut -d ',' -f 2- | sort -n -k1 -r"
             self.handle.sendline(command)
-            #print "ovs-vsctl Command sent status."
-            #self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
-            #print "sort return status: " 
-            #print self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
-            #print self.handle.before
+            self.handle.expect(["k1 -r",pexpect.EOF,pexpect.TIMEOUT])
             self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
-            #print "OFPST_FLOW expected status: " 
-            #print self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
-            #print self.handle.before
             response = self.handle.before
             #print "response=", response
-            return response 
-        
+            return response
+        else:
+            main.log.error("Unknown  protoVersion in get_flowTable(). given: ("+str(type(protoVersion))+") '"+str(protoVersion)+"'")
+
 
     def flow_comp(self,flow1,flow2):
-        #print "Inside flow compare function"
-        #print "Flow1 Table:"
-        #for flow in flow1:
-            #print flow1
-        #print "Flow2 Table"
-        #for flow in flow2:
-            #print flow2
-
         if flow1==flow2:
             return main.TRUE
         else:
             main.log.info("Flow tables do not match, printing tables:")
-            #main.log.info("Flow Table 1:")
-            #main.log.info(flow1)
-            #main.log.info("Flow Table 2:")
-            #main.log.info(flow2)
+            main.log.info("Flow Table 1:")
+            main.log.info(flow1)
+            main.log.info("Flow Table 2:")
+            main.log.info(flow2)
             return main.FALSE
 
     def setIpTablesOUTPUT(self, dst_ip, dst_port, action='add', packet_type='tcp',rule='DROP'):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index a290ac3..84c4b7f 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -693,7 +693,38 @@
             main.log.info(self.name+" ::::::")
             main.cleanup()
             main.exit()
-    
+
+    def roles_not_null(self):
+        '''
+        Iterates through each device and checks if there is a master assigned
+        Returns: main.TRUE if each device has a master
+                 main.FALSE any device has no master
+        '''
+        try:
+            import json
+            raw_roles = self.roles()
+            roles_json = json.loads(raw_roles)
+            #search json for the device with id then return the device
+            for device in roles_json:
+                #print device
+                if device['master'] == "none":
+                    main.log.warn("Device has no master: " + str(device) )
+                    return main.FALSE
+            return main.TRUE
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
+
+
     def paths(self, src_id, dst_id):
         '''
         Returns string of paths, and the cost.
@@ -1716,7 +1747,7 @@
         onos_node is the ip of one of the onos nodes in the cluster
         role must be either master, standby, or none
 
-        Returns main.TRUE or main.FALSE based argument varification.
+        Returns main.TRUE or main.FALSE based on argument verification.
             When device-role supports errors this should be extended to
             support that output
         '''
@@ -1808,5 +1839,127 @@
             main.cleanup()
             main.exit()
 
+    def election_test_leader(self):
+        '''
+         * CLI command to get the current leader for the Election test application.
+         #NOTE: Requires installation of the onos-app-election feature
+         Returns: Node IP of the leader if one exists
+                  None if none exists
+                  Main.FALSE on error
+        '''
+        try:
+            self.handle.sendline("election-test-leader")
+            self.handle.expect("election-test-leader")
+            self.handle.expect("onos>")
+            response = self.handle.before
+            #Leader
+            node_search = re.search("The\scurrent\sleader\sfor\sthe\sElection\sapp\sis\s(?P<node>.+)\.", response)
+            if node_search:
+                node = node_search.group('node')
+                main.log.info("Election-test-leader on "+str(self.name)+" found " + node + " as the leader")
+                return node
+            #no leader
+            null_search = re.search("There\sis\scurrently\sno\sleader\selected\sfor\sthe\sElection\sapp", response)
+            if null_search:
+                main.log.info("Election-test-leader found no leader on " + self.name )
+                return None
+            #error
+            if re.search("Command\snot\sfound", response):
+                main.log.error("Election app is not loaded on " + self.name)
+                return main.FALSE
+            else:
+                main.log.error("Error in election_test_leader: unexpected response")
+                main.log.error( repr(response) )
+                return main.FALSE
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
+
+    def election_test_run(self):
+        '''
+         * CLI command to run for leadership of the Election test application.
+         #NOTE: Requires installation of the onos-app-election feature
+         Returns: Main.TRUE on success
+                  Main.FALSE on error
+        '''
+        try:
+            self.handle.sendline("election-test-run")
+            self.handle.expect("election-test-run")
+            self.handle.expect("onos>")
+            response = self.handle.before
+            #success
+            search = re.search("Entering\sleadership\selections\sfor\sthe\sElection\sapp.", response)
+            if search:
+                main.log.info(self.name + " entering leadership elections for the Election app.")
+                return main.TRUE
+            #error
+            if re.search("Command\snot\sfound", response):
+                main.log.error("Election app is not loaded on " + self.name)
+                return main.FALSE
+            else:
+                main.log.error("Error in election_test_run: unexpected response")
+                main.log.error( repr(response) )
+                return main.FALSE
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
+
+    def election_test_withdraw(self):
+        '''
+         * CLI command to withdraw the local node from leadership election for
+         * the Election test application.
+         #NOTE: Requires installation of the onos-app-election feature
+         Returns: Main.TRUE on success
+                  Main.FALSE on error
+        '''
+        try:
+            self.handle.sendline("election-test-withdraw")
+            self.handle.expect("election-test-withdraw")
+            self.handle.expect("onos>")
+            response = self.handle.before
+            #success
+            search = re.search("Withdrawing\sfrom\sleadership\selections\sfor\sthe\sElection\sapp.", response)
+            if search:
+                main.log.info(self.name + " withdrawing from leadership elections for the Election app.")
+                return main.TRUE
+            #error
+            if re.search("Command\snot\sfound", response):
+                main.log.error("Election app is not loaded on " + self.name)
+                return main.FALSE
+            else:
+                main.log.error("Error in election_test_withdraw: unexpected response")
+                main.log.error( repr(response) )
+                return main.FALSE
+
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
 
     #***********************************
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 418554d..678bae4 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -1072,21 +1072,21 @@
                 if not line.startswith("time="):
                     continue
                 #else
-                print line
+                #print line
                 for var in line.split(","):
                     #print "'"+var+"'"
                     #print "'"+var.strip()+"'"
                     key, value = var.strip().split("=")
                     topology[key] = value
-            print "topology = ", topology
+            #print "topology = ", topology
             devices = topology.get('devices', False)
-            print "devices = ", devices
+            #print "devices = ", devices
             links = topology.get('links', False)
-            print "links = ", links
-            clusters = topology.get('clusters', False)
-            print "clusters = ", clusters
+            #print "links = ", links
+            SCCs = topology.get('SCC(s)', False)
+            #print "SCCs = ", SCCs
             paths = topology.get('paths', False)
-            print "paths = ", paths
+            #print "paths = ", paths
 
             return topology
         except pexpect.EOF:
@@ -1339,3 +1339,26 @@
             main.log.error( traceback.print_exc())
             main.log.info(self.name+" ::::::")
 
+    def check_logs(self, onos_ip):
+        '''
+        runs onos-check-logs on the given onos node
+        returns the response
+        '''
+        try:
+            cmd = "onos-check-logs " + str(onos_ip)
+            self.handle.sendline(cmd)
+            self.handle.expect(cmd)
+            self.handle.expect("\$")
+            response = self.handle.before
+            return response
+        except pexpect.EOF:
+            main.log.error("Lost ssh connection")
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+        except:
+            main.log.error("Some error in check_logs:")
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+
+
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 00d00ed..878839e 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -48,7 +48,7 @@
         ssh_newkey = 'Are you sure you want to continue connecting'
         refused = "ssh: connect to host "+self.ip_address+" port 22: Connection refused"
         if self.port:
-            self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address,maxread=50000)
+            self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address, env = {"TERM": "vt100"} , maxread=50000)
         else :
             self.handle =pexpect.spawn('ssh -X '+self.user_name+'@'+self.ip_address,maxread=1000000,timeout=60)
 
@@ -190,7 +190,9 @@
         '''
         ssh_newkey = 'Are you sure you want to continue connecting'
         refused = "ssh: connect to host "+ip_address+" port 22: Connection refused"
-        self.handle =pexpect.spawn('scp '+user_name+'@'+ip_address+':'+filepath+' '+dst_path)
+        cmd = 'scp '+str(user_name)+'@'+str(ip_address)+':'+str(filepath)+' '+str(dst_path)
+        main.log.info("Sending: " + cmd )
+        self.handle =pexpect.spawn( cmd )
         i=self.handle.expect([ssh_newkey,'password:',pexpect.EOF,pexpect.TIMEOUT,refused],120)
         
         if i==0:    
@@ -212,7 +214,9 @@
             main.log.error("ssh: connect to host "+ip_address+" port 22: Connection refused")
             return main.FALSE
 
-        self.handle.sendline("\r")
+        self.handle.sendline("")
+        self.handle.expect("$")
+        print self.handle.before
         
         return self.handle
     
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 0a8bd57..38a9b02 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -1,9 +1,10 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
     <cellName>HA</cellName>
     </ENV>
     <Git>False</Git>
+    <num_controllers> 7 </num_controllers>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
@@ -28,7 +29,7 @@
         <port7>6633</port7>
     </CTRL>
     <TESTONUSER>admin</TESTONUSER>
-    <TESTONIP>10.128.30.10</TESTONIP>
+    <TESTONIP>10.128.30.9</TESTONIP>
     <PING>
         <source1>h8</source1>
         <source2>h9</source2>
@@ -52,8 +53,8 @@
         <target10>10.0.0.27</target10>
     </PING>
     <timers>
-        <LinkDiscovery>2</LinkDiscovery>
-        <SwitchDiscovery>2</SwitchDiscovery>
+        <LinkDiscovery>.2</LinkDiscovery>
+        <SwitchDiscovery>.2</SwitchDiscovery>
     </timers>
     <MNtcpdump>
         <intf>eth0</intf>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 47dc744..b9a07b9 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -16,6 +16,8 @@
 CASE11: Switch down
 CASE12: Switch up
 CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
 '''
 class HATestClusterRestart:
 
@@ -61,6 +63,7 @@
         global ONOS6_port
         global ONOS7_ip
         global ONOS7_port
+        global num_controllers
 
         ONOS1_ip = main.params['CTRL']['ip1']
         ONOS1_port = main.params['CTRL']['port1']
@@ -76,13 +79,14 @@
         ONOS6_port = main.params['CTRL']['port6']
         ONOS7_ip = main.params['CTRL']['ip7']
         ONOS7_port = main.params['CTRL']['port7']
+        num_controllers = int(main.params['num_controllers'])
 
 
         main.step("Applying cell variable to environment")
         cell_result = main.ONOSbench.set_cell(cell_name)
         verify_result = main.ONOSbench.verify_cell()
 
-        #FIXME:this is short term fix 
+        #FIXME:this is short term fix
         main.log.report("Removing raft logs")
         main.ONOSbench.onos_remove_raft_logs()
         main.log.report("Uninstalling ONOS")
@@ -143,31 +147,33 @@
 
 
         main.step("Checking if ONOS is up yet")
-        #TODO: Refactor
-        # check bundle:list?
-        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
-        if not onos1_isup:
-            main.log.report("ONOS1 didn't start!")
-        onos2_isup = main.ONOSbench.isup(ONOS2_ip)
-        if not onos2_isup:
-            main.log.report("ONOS2 didn't start!")
-        onos3_isup = main.ONOSbench.isup(ONOS3_ip)
-        if not onos3_isup:
-            main.log.report("ONOS3 didn't start!")
-        onos4_isup = main.ONOSbench.isup(ONOS4_ip)
-        if not onos4_isup:
-            main.log.report("ONOS4 didn't start!")
-        onos5_isup = main.ONOSbench.isup(ONOS5_ip)
-        if not onos5_isup:
-            main.log.report("ONOS5 didn't start!")
-        onos6_isup = main.ONOSbench.isup(ONOS6_ip)
-        if not onos6_isup:
-            main.log.report("ONOS6 didn't start!")
-        onos7_isup = main.ONOSbench.isup(ONOS7_ip)
-        if not onos7_isup:
-            main.log.report("ONOS7 didn't start!")
-        onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
-                and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+        #TODO check bundle:list?
+        for i in range(2):
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            if not onos1_isup:
+                main.log.report("ONOS1 didn't start!")
+            onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+            if not onos2_isup:
+                main.log.report("ONOS2 didn't start!")
+            onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+            if not onos3_isup:
+                main.log.report("ONOS3 didn't start!")
+            onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+            if not onos4_isup:
+                main.log.report("ONOS4 didn't start!")
+            onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+            if not onos5_isup:
+                main.log.report("ONOS5 didn't start!")
+            onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+            if not onos6_isup:
+                main.log.report("ONOS6 didn't start!")
+            onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+            if not onos7_isup:
+                main.log.report("ONOS7 didn't start!")
+            onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+                    and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+            if onos_isup_result == main.TRUE:
+                break
         # TODO: if it becomes an issue, we can retry this step  a few times
 
 
@@ -197,9 +203,9 @@
                 onfail="Test startup NOT successful")
 
 
-        #if case1_result==main.FALSE:
-        #    main.cleanup()
-        #    main.exit()
+        if case1_result==main.FALSE:
+            main.cleanup()
+            main.exit()
 
     def CASE2(self,main) :
         '''
@@ -209,31 +215,12 @@
         import json
         import re
 
-
-        '''
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS1_port = main.params['CTRL']['port1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS2_port = main.params['CTRL']['port2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS3_port = main.params['CTRL']['port3']
-        ONOS4_ip = main.params['CTRL']['ip4']
-        ONOS4_port = main.params['CTRL']['port4']
-        ONOS5_ip = main.params['CTRL']['ip5']
-        ONOS5_port = main.params['CTRL']['port5']
-        ONOS6_ip = main.params['CTRL']['ip6']
-        ONOS6_port = main.params['CTRL']['port6']
-        ONOS7_ip = main.params['CTRL']['ip7']
-        ONOS7_port = main.params['CTRL']['port7']
-        '''
-
-
         main.log.report("Assigning switches to controllers")
         main.case("Assigning Controllers")
         main.step("Assign switches to controllers")
 
         for i in range (1,29):
-           main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+           main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
                     ip1=ONOS1_ip,port1=ONOS1_port,
                     ip2=ONOS2_ip,port2=ONOS2_port,
                     ip3=ONOS3_ip,port3=ONOS3_port,
@@ -265,8 +252,217 @@
                 onpass="Switch mastership assigned correctly",
                 onfail="Switches not assigned correctly to controllers")
 
-        #TODO: If assign roles is working reliably then manually 
-        #   assign mastership to the controller we want
+        #Manually assign mastership to the controller we want
+        role_call = main.TRUE
+        role_check = main.TRUE
+
+        device_id = main.ONOScli1.get_device("1000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2800")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("5000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id =  main.ONOScli1.get_device("3004")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+        if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3008")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3009")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3010")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3011")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3012")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3013")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3014")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3015")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3016")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3017")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6007")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+        if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6018")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6019")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6020")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6021")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6022")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6023")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6024")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6025")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6026")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6027")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_call,
+                onpass="Re-assigned switch mastership to designated controller",
+                onfail="Something wrong with device_role calls")
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_check,
+                onpass="Switches were successfully reassigned to designated controller",
+                onfail="Switches were not successfully reassigned")
+        mastership_check = mastership_check and role_call and role_check
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
 
 
     def CASE3(self,main) :
@@ -274,6 +470,7 @@
         Assign intents
 
         """
+        #FIXME: we must reinstall intents until we have a persistant datastore!
         import time
         import json
         import re
@@ -283,6 +480,16 @@
         main.step("Discovering  Hosts( Via pingall for now)")
         #FIXME: Once we have a host discovery mechanism, use that instead
 
+        #install onos-app-fwd
+        main.log.info("Install reactive forwarding app")
+        main.ONOScli1.feature_install("onos-app-fwd")
+        main.ONOScli2.feature_install("onos-app-fwd")
+        main.ONOScli3.feature_install("onos-app-fwd")
+        main.ONOScli4.feature_install("onos-app-fwd")
+        main.ONOScli5.feature_install("onos-app-fwd")
+        main.ONOScli6.feature_install("onos-app-fwd")
+        main.ONOScli7.feature_install("onos-app-fwd")
+
         #REACTIVE FWD test
         ping_result = main.FALSE
         time1 = time.time()
@@ -299,22 +506,30 @@
         main.ONOScli5.feature_uninstall("onos-app-fwd")
         main.ONOScli6.feature_uninstall("onos-app-fwd")
         main.ONOScli7.feature_uninstall("onos-app-fwd")
+        #timeout for fwd flows
+        time.sleep(10)
 
         main.step("Add  host intents")
         #TODO:  move the host numbers to params
         import json
         intents_json= json.loads(main.ONOScli1.hosts())
-        intent_add_result = main.FALSE
+        intent_add_result = True
         for i in range(8,18):
             main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
             host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
             host2 =  "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
-            #NOTE: get host can return None
-            #TODO: handle this
             host1_id = main.ONOScli1.get_host(host1)['id']
             host2_id = main.ONOScli1.get_host(host2)['id']
-            tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
-            intent_add_result = intent_add_result and tmp_result
+            #NOTE: get host can return None
+            if host1_id and host2_id:
+                tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+            else:
+                main.log.error("Error, get_host() failed")
+                tmp_result = main.FALSE
+            intent_add_result = bool(intent_add_result and tmp_result)
+        utilities.assert_equals(expect=True, actual=intent_add_result,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
         #TODO Check if intents all exist in datastore
         #NOTE: Do we need to print this once the test is working?
         #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -360,6 +575,25 @@
 
         main.step("Get the Mastership of each switch from each controller")
         global mastership_state
+        mastership_state = []
+
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
         ONOS3_mastership = main.ONOScli3.roles()
@@ -416,6 +650,7 @@
 
         main.step("Get the intents from each controller")
         global intent_state
+        intent_state = []
         ONOS1_intents = main.ONOScli1.intents( json_format=True )
         ONOS2_intents = main.ONOScli2.intents( json_format=True )
         ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -470,6 +705,7 @@
 
         main.step("Get the flows from each controller")
         global flow_state
+        flow_state = []
         ONOS1_flows = main.ONOScli1.flows( json_format=True )
         ONOS2_flows = main.ONOScli2.flows( json_format=True )
         ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -527,7 +763,7 @@
         global flows
         flows=[]
         for i in range(1,29):
-            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+            flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
 
         #TODO: Compare switch flow tables with ONOS flow tables
 
@@ -578,7 +814,6 @@
         devices.append( main.ONOScli5.devices() )
         devices.append( main.ONOScli6.devices() )
         devices.append( main.ONOScli7.devices() )
-        '''
         hosts = []
         hosts.append( main.ONOScli1.hosts() )
         hosts.append( main.ONOScli2.hosts() )
@@ -587,7 +822,6 @@
         hosts.append( main.ONOScli5.hosts() )
         hosts.append( main.ONOScli6.hosts() )
         hosts.append( main.ONOScli7.hosts() )
-        '''
         ports = []
         ports.append( main.ONOScli1.ports() )
         ports.append( main.ONOScli2.ports() )
@@ -604,13 +838,98 @@
         links.append( main.ONOScli5.links() )
         links.append( main.ONOScli6.links() )
         links.append( main.ONOScli7.links() )
+        clusters = []
+        clusters.append( main.ONOScli1.clusters() )
+        clusters.append( main.ONOScli2.clusters() )
+        clusters.append( main.ONOScli3.clusters() )
+        clusters.append( main.ONOScli4.clusters() )
+        clusters.append( main.ONOScli5.clusters() )
+        clusters.append( main.ONOScli6.clusters() )
+        clusters.append( main.ONOScli7.clusters() )
+        paths = []
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+        paths.append( temp_topo.get('paths', False) )
+
+        #Compare json objects for hosts, dataplane clusters and paths
+
+        #hosts
+        consistent_hosts_result = main.TRUE
+        for controller in range( len( hosts ) ):
+            if not "Error" in hosts[controller]:
+                if hosts[controller] == hosts[0]:
+                    continue
+                else:#hosts not consistent
+                    main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    main.log.warn( repr( hosts[controller] ) )
+                    consistent_hosts_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                consistent_hosts_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                onpass="Hosts view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of hosts")
+
+        #Strongly connected clusters of devices
+        consistent_clusters_result = main.TRUE
+        for controller in range( len( clusters ) ):
+            if not "Error" in clusters[controller]:
+                if clusters[controller] == clusters[0]:
+                    continue
+                else:#clusters not consistent
+                    main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_clusters_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                consistent_clusters_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                onpass="Clusters view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of clusters")
+        num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+        utilities.assert_equals(expect = 1, actual = num_clusters,
+                onpass="ONOS shows 1 SCC",
+                onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+        #paths
+        consistent_paths_result = main.TRUE
+        for controller in range( len( paths ) ):
+            if not "Error" in paths[controller]:
+                if paths[controller] == paths[0]:
+                    continue
+                else:#paths not consistent
+                    main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_paths_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                consistent_paths_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                onpass="Paths count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different counts of paths")
 
 
         main.step("Comparing ONOS topology to MN")
         devices_results = main.TRUE
         ports_results = main.TRUE
         links_results = main.TRUE
-        for controller in range(7): #TODO parameterize the number of controllers
+        for controller in range(num_controllers):
             if devices[controller] or not "Error" in devices[controller]:
                 current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
             else:
@@ -639,14 +958,16 @@
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
 
-        topo_result = devices_results and ports_results and links_results
+        topo_result = devices_results and ports_results and links_results\
+                and consistent_hosts_result and consistent_clusters_result\
+                and consistent_paths_result
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
                 onfail="Topology Check Test NOT successful")
 
         final_assert = main.TRUE
         final_assert = final_assert and topo_result and flow_check \
-                and intent_check and consistent_mastership
+                and intent_check and consistent_mastership and roles_not_null
         utilities.assert_equals(expect=main.TRUE, actual=final_assert,
                 onpass="State check successful",
                 onfail="State check NOT successful")
@@ -708,6 +1029,24 @@
         import json
         main.case("Running ONOS Constant State Tests")
 
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
+
         main.step("Check if switch roles are consistent across all nodes")
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
@@ -739,7 +1078,6 @@
                 and ONOS1_mastership == ONOS5_mastership\
                 and ONOS1_mastership == ONOS6_mastership\
                 and ONOS1_mastership == ONOS7_mastership:
-                    #mastership_state = ONOS1_mastership
                     consistent_mastership = main.TRUE
                     main.log.report("Switch roles are consistent across all ONOS nodes")
         else:
@@ -785,7 +1123,7 @@
                 onpass="Mastership of Switches was not changed",
                 onfail="Mastership of some switches changed")
         #NOTE: we expect mastership to change on controller failure
-        mastership_check = mastership_check #and consistent_mastership
+        mastership_check = consistent_mastership
 
 
 
@@ -822,35 +1160,53 @@
                     intent_check = main.TRUE
                     main.log.report("Intents are consistent across all ONOS nodes")
         else:
-            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS1 intents: ")
+            print json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS2 intents: ")
+            print json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS3 intents: ")
+            print json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS4 intents: ")
+            print json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS5 intents: ")
+            print json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS6 intents: ")
+            print json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS7 intents: ")
+            print json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
         utilities.assert_equals(expect = main.TRUE,actual=intent_check,
                 onpass="Intents are consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of intents")
 
+        #NOTE: Hazelcast has no durability, so intents are lost across system restarts
+        '''
         main.step("Compare current intents with intents before the failure")
+        #NOTE: this requires case 5 to pass for intent_state to be set.
+        #      maybe we should stop the test if that fails?
         if intent_state == ONOS1_intents:
             same_intents = main.TRUE
             main.log.report("Intents are consistent with before failure")
         #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
         else:
+            try:
+                main.log.warn("ONOS1 intents: ")
+                print json.dumps(json.loads(ONOS1_intents),
+                    sort_keys=True, indent=4, separators=(',', ': '))
+            except:
+                pass
             same_intents = main.FALSE
         utilities.assert_equals(expect = main.TRUE,actual=same_intents,
                 onpass="Intents are consistent with before failure",
                 onfail="The Intents changed during failure")
         intent_check = intent_check and same_intents
+        '''
 
 
 
@@ -859,12 +1215,12 @@
         flows2=[]
         for i in range(28):
             main.log.info("Checking flow table on s" + str(i+1))
-            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
             flows2.append(tmp_flows)
-            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            Flow_Tables = Flow_Tables and temp_result
             if Flow_Tables == main.FALSE:
                 main.log.info("Differences in flow table for switch: "+str(i+1))
-                break
         if Flow_Tables == main.TRUE:
             main.log.report("No changes were found in the flow tables")
         utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -880,7 +1236,7 @@
         #NOTE: checkForLoss returns main.FALSE with 0% packet loss
         for i in range(8,18):
             main.log.info("Checking for a loss in pings along flow from s" + str(i))
-            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+            Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
         if Loss_In_Pings == main.TRUE:
             main.log.info("Loss in ping detected")
         elif Loss_In_Pings == main.ERROR:
@@ -891,10 +1247,37 @@
         utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
                 onpass="No Loss of connectivity",
                 onfail="Loss of dataplane connectivity detected")
+        #NOTE: Since intents are not persisted with Hazelcast, we expect this
+        Loss_In_Pings = main.FALSE
 
 
-        #TODO:add topology to this or leave as a seperate case?
-        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        #Test of LeadershipElection
+        leader_list = []
+        leader_result = main.TRUE
+        for controller in range(1,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            leaderN = node.election_test_leader()
+            leader_list.append(leaderN)
+            if leaderN == main.FALSE:
+                #error in  response
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+            elif leaderN == None:
+                main.log.report("ONOS"+str(controller) + " shows no leader for the election-app was elected after the old one died")
+                leader_result = main.FALSE
+        if len( set( leader_list ) ) != 1:
+            leader_result = main.FALSE
+            main.log.error("Inconsistent view of leader for the election test app")
+            #TODO: print the list
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected after restart)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+                and leader_result
         result = int(result)
         if result == main.TRUE:
             main.log.report("Constant State Tests Passed")
@@ -936,87 +1319,173 @@
         ports_results = main.TRUE
         links_results = main.TRUE
         topo_result = main.FALSE
-        start_time = time.time()
         elapsed = 0
         count = 0
-        while topo_result == main.FALSE and elapsed < 120:
+        main.step("Collecting topology information from ONOS")
+        start_time = time.time()
+        while topo_result == main.FALSE and elapsed < 60:
             count = count + 1
-            try:
-                main.step("Collecting topology information from ONOS")
-                devices = []
-                devices.append( main.ONOScli1.devices() )
-                devices.append( main.ONOScli2.devices() )
-                devices.append( main.ONOScli3.devices() )
-                devices.append( main.ONOScli4.devices() )
-                devices.append( main.ONOScli5.devices() )
-                devices.append( main.ONOScli6.devices() )
-                devices.append( main.ONOScli7.devices() )
-                '''
-                hosts = []
-                hosts.append( main.ONOScli1.hosts() )
-                hosts.append( main.ONOScli2.hosts() )
-                hosts.append( main.ONOScli3.hosts() )
-                hosts.append( main.ONOScli4.hosts() )
-                hosts.append( main.ONOScli5.hosts() )
-                hosts.append( main.ONOScli6.hosts() )
-                hosts.append( main.ONOScli7.hosts() )
-                '''
-                ports = []
-                ports.append( main.ONOScli1.ports() )
-                ports.append( main.ONOScli2.ports() )
-                ports.append( main.ONOScli3.ports() )
-                ports.append( main.ONOScli4.ports() )
-                ports.append( main.ONOScli5.ports() )
-                ports.append( main.ONOScli6.ports() )
-                ports.append( main.ONOScli7.ports() )
-                links = []
-                links.append( main.ONOScli1.links() )
-                links.append( main.ONOScli2.links() )
-                links.append( main.ONOScli3.links() )
-                links.append( main.ONOScli4.links() )
-                links.append( main.ONOScli5.links() )
-                links.append( main.ONOScli6.links() )
-                links.append( main.ONOScli7.links() )
+            if count > 1:
+                MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+            cli_start = time.time()
+            devices = []
+            devices.append( main.ONOScli1.devices() )
+            devices.append( main.ONOScli2.devices() )
+            devices.append( main.ONOScli3.devices() )
+            devices.append( main.ONOScli4.devices() )
+            devices.append( main.ONOScli5.devices() )
+            devices.append( main.ONOScli6.devices() )
+            devices.append( main.ONOScli7.devices() )
+            hosts = []
+            hosts.append( main.ONOScli1.hosts() )
+            hosts.append( main.ONOScli2.hosts() )
+            hosts.append( main.ONOScli3.hosts() )
+            hosts.append( main.ONOScli4.hosts() )
+            hosts.append( main.ONOScli5.hosts() )
+            hosts.append( main.ONOScli6.hosts() )
+            hosts.append( main.ONOScli7.hosts() )
+            ports = []
+            ports.append( main.ONOScli1.ports() )
+            ports.append( main.ONOScli2.ports() )
+            ports.append( main.ONOScli3.ports() )
+            ports.append( main.ONOScli4.ports() )
+            ports.append( main.ONOScli5.ports() )
+            ports.append( main.ONOScli6.ports() )
+            ports.append( main.ONOScli7.ports() )
+            links = []
+            links.append( main.ONOScli1.links() )
+            links.append( main.ONOScli2.links() )
+            links.append( main.ONOScli3.links() )
+            links.append( main.ONOScli4.links() )
+            links.append( main.ONOScli5.links() )
+            links.append( main.ONOScli6.links() )
+            links.append( main.ONOScli7.links() )
+            clusters = []
+            clusters.append( main.ONOScli1.clusters() )
+            clusters.append( main.ONOScli2.clusters() )
+            clusters.append( main.ONOScli3.clusters() )
+            clusters.append( main.ONOScli4.clusters() )
+            clusters.append( main.ONOScli5.clusters() )
+            clusters.append( main.ONOScli6.clusters() )
+            clusters.append( main.ONOScli7.clusters() )
+            paths = []
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+            paths.append( temp_topo.get('paths', False) )
 
-                for controller in range(7): #TODO parameterize the number of controllers
-                    if devices[controller] or not "Error" in devices[controller]:
-                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
-                    else:
-                        current_devices_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
-                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
-                    if ports[controller] or not "Error" in ports[controller]:
-                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
-                    else:
-                        current_ports_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
-                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+            elapsed = time.time() - start_time
+            cli_time = time.time() - cli_start
+            print "CLI time: " + str(cli_time)
 
-                    if links[controller] or not "Error" in links[controller]:
-                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
-                    else:
-                        current_links_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
-                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
-            except:
-                main.log.error("something went wrong in topo comparison")
-                main.log.warn( repr( devices ) )
-                main.log.warn( repr( ports ) )
-                main.log.warn( repr( links ) )
+            for controller in range(num_controllers):
+                if devices[controller] or not "Error" in devices[controller]:
+                    current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                else:
+                    current_devices_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                        onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
+                if ports[controller] or not "Error" in ports[controller]:
+                    current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                else:
+                    current_ports_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                        onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                if links[controller] or not "Error" in links[controller]:
+                    current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                else:
+                    current_links_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                        onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
             devices_results = devices_results and current_devices_result
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
-            topo_result = devices_results and ports_results and links_results
-            elapsed = time.time() - start_time
-        time_threshold = elapsed < 1
-        topo_result = topo_result and time_threshold
-        #TODO make sure this step is non-blocking. IE add a timeout
-        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+            #Compare json objects for hosts, dataplane clusters and paths
+
+            #hosts
+            consistent_hosts_result = main.TRUE
+            for controller in range( len( hosts ) ):
+                if not "Error" in hosts[controller]:
+                    if hosts[controller] == hosts[0]:
+                        continue
+                    else:#hosts not consistent
+                        main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        main.log.warn( repr( hosts[controller] ) )
+                        consistent_hosts_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                    consistent_hosts_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                    onpass="Hosts view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of hosts")
+
+            #Strongly connected clusters of devices
+            consistent_clusters_result = main.TRUE
+            for controller in range( len( clusters ) ):
+                if not "Error" in clusters[controller]:
+                    if clusters[controller] == clusters[0]:
+                        continue
+                    else:#clusters not consistent
+                        main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_clusters_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                    consistent_clusters_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                    onpass="Clusters view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of clusters")
+            num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+            utilities.assert_equals(expect = 1, actual = num_clusters,
+                    onpass="ONOS shows 1 SCC",
+                    onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+            #paths
+            consistent_paths_result = main.TRUE
+            for controller in range( len( paths ) ):
+                if not "Error" in paths[controller]:
+                    if paths[controller] == paths[0]:
+                        continue
+                    else:#paths not consistent
+                        main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_paths_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                    consistent_paths_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                    onpass="Paths count is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different counts of paths")
+
+
+            topo_result = devices_results and ports_results and links_results\
+                    and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+        topo_result = topo_result and int(count <= 2)
+        note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+        main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
                 str(elapsed) + " seconds, " + str(count) +" tries" )
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
@@ -1031,7 +1500,7 @@
         '''
         #NOTE: You should probably run a topology check after this
 
-        link_sleep = int(main.params['timers']['LinkDiscovery'])
+        link_sleep = float(main.params['timers']['LinkDiscovery'])
 
         description = "Turn off a link to ensure that Link Discovery is working properly"
         main.log.report(description)
@@ -1053,7 +1522,7 @@
         '''
         #NOTE: You should probably run a topology check after this
 
-        link_sleep = int(main.params['timers']['LinkDiscovery'])
+        link_sleep = float(main.params['timers']['LinkDiscovery'])
 
         description = "Restore a link to ensure that Link Discovery is working properly"
         main.log.report(description)
@@ -1076,7 +1545,7 @@
         #NOTE: You should probably run a topology check after this
         import time
 
-        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+        switch_sleep = float(main.params['timers']['SwitchDiscovery'])
 
         description = "Killing a switch to ensure it is discovered correctly"
         main.log.report(description)
@@ -1085,13 +1554,18 @@
         #TODO: Make this switch parameterizable
         main.step("Kill s28 ")
         main.log.report("Deleting s28")
-        #FIXME: use new dynamic topo functions
         main.Mininet1.del_switch("s28")
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
         time.sleep(switch_sleep)
+        device = main.ONOScli1.get_device(dpid="0028")
         #Peek at the deleted switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == False:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Kill switch succesful",
+                onfail="Failed to kill switch?")
 
     def CASE12 (self, main) :
         '''
@@ -1099,7 +1573,8 @@
         '''
         #NOTE: You should probably run a topology check after this
         import time
-        #FIXME: use new dynamic topo functions
+
+        switch_sleep = float(main.params['timers']['SwitchDiscovery'])
         description = "Adding a switch to ensure it is discovered correctly"
         main.log.report(description)
         main.case(description)
@@ -1111,7 +1586,7 @@
         main.Mininet1.add_link('s28', 's3')
         main.Mininet1.add_link('s28', 's6')
         main.Mininet1.add_link('s28', 'h28')
-        main.Mininet1.assign_sw_controller(sw="28",count=7,
+        main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
                 ip1=ONOS1_ip,port1=ONOS1_port,
                 ip2=ONOS2_ip,port2=ONOS2_port,
                 ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1121,9 +1596,15 @@
                 ip7=ONOS7_ip,port7=ONOS7_port)
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
         time.sleep(switch_sleep)
-        #Peek at the added switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        device = main.ONOScli1.get_device(dpid="0028")
+        #Peek at the deleted switch
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == True:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="add switch succesful",
+                onfail="Failed to add switch?")
 
     def CASE13 (self, main) :
         '''
@@ -1131,14 +1612,41 @@
         '''
         import os
         import time
+        #printing colors to terminal
+        colors = {}
+        colors['cyan']   = '\033[96m'
+        colors['purple'] = '\033[95m'
+        colors['blue']   = '\033[94m'
+        colors['green']  = '\033[92m'
+        colors['yellow'] = '\033[93m'
+        colors['red']    = '\033[91m'
+        colors['end']    = '\033[0m'
         description = "Test Cleanup"
         main.log.report(description)
         main.case(description)
         main.step("Killing tcpdumps")
         main.Mininet2.stop_tcpdump()
 
+        main.step("Checking ONOS Logs for errors")
+        print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS1_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS2_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS3_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS4_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS5_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS6_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS7_ip)
+
         main.step("Copying MN pcap and ONOS log files to test station")
         testname = main.TEST
+        teststation_user = main.params['TESTONUSER']
+        teststation_IP = main.params['TESTONIP']
         #NOTE: MN Pcap file is being saved to ~/packet_captures
         #       scp this file as MN and TestON aren't necessarily the same vm
         #FIXME: scp
@@ -1150,19 +1658,26 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
         #std*.log's
@@ -1172,24 +1687,29 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
 
-
-
         #sleep so scp can finish
         time.sleep(10)
         main.step("Packing and rotating pcap archives")
@@ -1200,3 +1720,139 @@
         utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
                 onpass="Test cleanup successful",
                 onfail="Test cleanup NOT successful")
+
+    def CASE14 ( self, main ) :
+        '''
+        start election app on all onos nodes
+        '''
+        leader_result = main.TRUE
+        #install app on onos 1
+        main.log.info("Install leadership election app")
+        main.ONOScli1.feature_install("onos-app-election")
+        #wait for election
+        #check for leader
+        leader = main.ONOScli1.election_test_leader()
+        #verify leader is ONOS1
+        if leader == ONOS1_ip:
+            #all is well
+            pass
+        elif leader == None:
+            #No leader elected
+            main.log.report("No leader was elected")
+            leader_result = main.FALSE
+        elif leader == main.FALSE:
+            #error in  response
+            #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+            main.log.report("Something is wrong with election_test_leader function, check the error logs")
+            leader_result = main.FALSE
+        else:
+            #error in  response
+            main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+            leader_result = main.FALSE
+
+
+
+
+        #install on other nodes and check for leader.
+        #Should be onos1 and each app should show the same leader
+        for controller in range(2,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            node.feature_install("onos-app-election")
+            leaderN = node.election_test_leader()
+            #verify leader is ONOS1
+            if leaderN == ONOS1_ip:
+                #all is well
+                pass
+            elif leaderN == main.FALSE:
+                #error in  response
+                #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+            elif leader != leaderN:
+                leader_result = main.FALSE
+                main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+                        " as the leader of the election app. Leader should be "+str(leader) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+    def CASE15 ( self, main ) :
+        '''
+        Check that Leadership Election is still functional
+        '''
+        leader_result = main.TRUE
+        description = "Check that Leadership Election is still functional"
+        main.log.report(description)
+        main.case(description)
+        main.step("Find current leader and withdraw")
+        leader = main.ONOScli1.election_test_leader()
+        #TODO: do some sanity checking on leader before using it
+        withdraw_result = main.FALSE
+        if leader == ONOS1_ip:
+            old_leader = getattr( main, "ONOScli1" )
+        elif leader == ONOS2_ip:
+            old_leader = getattr( main, "ONOScli2" )
+        elif leader == ONOS3_ip:
+            old_leader = getattr( main, "ONOScli3" )
+        elif leader == ONOS4_ip:
+            old_leader = getattr( main, "ONOScli4" )
+        elif leader == ONOS5_ip:
+            old_leader = getattr( main, "ONOScli5" )
+        elif leader == ONOS6_ip:
+            old_leader = getattr( main, "ONOScli6" )
+        elif leader == ONOS7_ip:
+            old_leader = getattr( main, "ONOScli7" )
+        elif leader == None or leader == main.FALSE:
+            main.log.report("Leader for the election app should be an ONOS node,"\
+                    +"instead got '"+str(leader)+"'")
+            leader_result = main.FALSE
+        withdraw_result = old_leader.election_test_withdraw()
+
+
+        main.step("Make sure new leader is elected")
+        leader_list = []
+        for controller in range(1,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            leader_list.append( node.election_test_leader() )
+        for leaderN in leader_list:
+            if leaderN == leader:
+                main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+                        " as leader after they withdrew")
+                leader_result = main.FALSE
+            elif leaderN == main.FALSE:
+                #error in  response
+                #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+        consistent_leader = main.FALSE
+        if len( set( leader_list ) ) == 1:
+            main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+            consistent_leader = main.TRUE
+        else:
+            main.log.report("Inconsistent responses for leader of Election-app:")
+            for n in range(len(leader_list)):
+                main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+
+        main.step("Run for election on old leader(just so everyone is in the hat)")
+        run_result = old_leader.election_test_run()
+        if consistent_leader == main.TRUE:
+            after_run = main.ONOScli1.election_test_leader()
+            #verify leader didn't just change
+            if after_run == leader_list[0]:
+                leader_result = main.TRUE
+            else:
+                leader_result = main.FALSE
+        #TODO: assert on  run and withdraw results?
+
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index 0a8bd57..78e0c83 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -1,9 +1,10 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
     <cellName>HA</cellName>
     </ENV>
     <Git>False</Git>
+    <num_controllers> 7 </num_controllers>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
@@ -28,7 +29,7 @@
         <port7>6633</port7>
     </CTRL>
     <TESTONUSER>admin</TESTONUSER>
-    <TESTONIP>10.128.30.10</TESTONIP>
+    <TESTONIP>10.128.30.9</TESTONIP>
     <PING>
         <source1>h8</source1>
         <source2>h9</source2>
@@ -52,8 +53,8 @@
         <target10>10.0.0.27</target10>
     </PING>
     <timers>
-        <LinkDiscovery>2</LinkDiscovery>
-        <SwitchDiscovery>2</SwitchDiscovery>
+        <LinkDiscovery>.2</LinkDiscovery>
+        <SwitchDiscovery>.2</SwitchDiscovery>
     </timers>
     <MNtcpdump>
         <intf>eth0</intf>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 2af775d..b9d2925 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -16,6 +16,8 @@
 CASE11: Switch down
 CASE12: Switch up
 CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
 '''
 class HATestMinorityRestart:
 
@@ -61,6 +63,7 @@
         global ONOS6_port
         global ONOS7_ip
         global ONOS7_port
+        global num_controllers
 
         ONOS1_ip = main.params['CTRL']['ip1']
         ONOS1_port = main.params['CTRL']['port1']
@@ -76,13 +79,14 @@
         ONOS6_port = main.params['CTRL']['port6']
         ONOS7_ip = main.params['CTRL']['ip7']
         ONOS7_port = main.params['CTRL']['port7']
+        num_controllers = int(main.params['num_controllers'])
 
 
         main.step("Applying cell variable to environment")
         cell_result = main.ONOSbench.set_cell(cell_name)
         verify_result = main.ONOSbench.verify_cell()
 
-        #FIXME:this is short term fix 
+        #FIXME:this is short term fix
         main.log.report("Removing raft logs")
         main.ONOSbench.onos_remove_raft_logs()
         main.log.report("Uninstalling ONOS")
@@ -143,31 +147,33 @@
 
 
         main.step("Checking if ONOS is up yet")
-        #TODO: Refactor
-        # check bundle:list?
-        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
-        if not onos1_isup:
-            main.log.report("ONOS1 didn't start!")
-        onos2_isup = main.ONOSbench.isup(ONOS2_ip)
-        if not onos2_isup:
-            main.log.report("ONOS2 didn't start!")
-        onos3_isup = main.ONOSbench.isup(ONOS3_ip)
-        if not onos3_isup:
-            main.log.report("ONOS3 didn't start!")
-        onos4_isup = main.ONOSbench.isup(ONOS4_ip)
-        if not onos4_isup:
-            main.log.report("ONOS4 didn't start!")
-        onos5_isup = main.ONOSbench.isup(ONOS5_ip)
-        if not onos5_isup:
-            main.log.report("ONOS5 didn't start!")
-        onos6_isup = main.ONOSbench.isup(ONOS6_ip)
-        if not onos6_isup:
-            main.log.report("ONOS6 didn't start!")
-        onos7_isup = main.ONOSbench.isup(ONOS7_ip)
-        if not onos7_isup:
-            main.log.report("ONOS7 didn't start!")
-        onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
-                and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+        #TODO check bundle:list?
+        for i in range(2):
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            if not onos1_isup:
+                main.log.report("ONOS1 didn't start!")
+            onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+            if not onos2_isup:
+                main.log.report("ONOS2 didn't start!")
+            onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+            if not onos3_isup:
+                main.log.report("ONOS3 didn't start!")
+            onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+            if not onos4_isup:
+                main.log.report("ONOS4 didn't start!")
+            onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+            if not onos5_isup:
+                main.log.report("ONOS5 didn't start!")
+            onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+            if not onos6_isup:
+                main.log.report("ONOS6 didn't start!")
+            onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+            if not onos7_isup:
+                main.log.report("ONOS7 didn't start!")
+            onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+                    and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+            if onos_isup_result == main.TRUE:
+                break
         # TODO: if it becomes an issue, we can retry this step  a few times
 
 
@@ -181,6 +187,7 @@
         cli_results = cli_result1 and cli_result2 and cli_result3 and\
                 cli_result4 and cli_result5 and cli_result6 and cli_result7
 
+
         main.step("Start Packet Capture MN")
         main.Mininet2.start_tcpdump(
                 str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
@@ -197,9 +204,9 @@
                 onfail="Test startup NOT successful")
 
 
-        #if case1_result==main.FALSE:
-        #    main.cleanup()
-        #    main.exit()
+        if case1_result==main.FALSE:
+            main.cleanup()
+            main.exit()
 
     def CASE2(self,main) :
         '''
@@ -209,31 +216,12 @@
         import json
         import re
 
-
-        '''
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS1_port = main.params['CTRL']['port1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS2_port = main.params['CTRL']['port2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS3_port = main.params['CTRL']['port3']
-        ONOS4_ip = main.params['CTRL']['ip4']
-        ONOS4_port = main.params['CTRL']['port4']
-        ONOS5_ip = main.params['CTRL']['ip5']
-        ONOS5_port = main.params['CTRL']['port5']
-        ONOS6_ip = main.params['CTRL']['ip6']
-        ONOS6_port = main.params['CTRL']['port6']
-        ONOS7_ip = main.params['CTRL']['ip7']
-        ONOS7_port = main.params['CTRL']['port7']
-        '''
-
-
         main.log.report("Assigning switches to controllers")
         main.case("Assigning Controllers")
         main.step("Assign switches to controllers")
 
         for i in range (1,29):
-           main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+           main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
                     ip1=ONOS1_ip,port1=ONOS1_port,
                     ip2=ONOS2_ip,port2=ONOS2_port,
                     ip3=ONOS3_ip,port3=ONOS3_port,
@@ -265,8 +253,217 @@
                 onpass="Switch mastership assigned correctly",
                 onfail="Switches not assigned correctly to controllers")
 
-        #TODO: If assign roles is working reliably then manually 
-        #   assign mastership to the controller we want
+        #Manually assign mastership to the controller we want
+        role_call = main.TRUE
+        role_check = main.TRUE
+
+        device_id = main.ONOScli1.get_device("1000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2800")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("5000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id =  main.ONOScli1.get_device("3004")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+        if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3008")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3009")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3010")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3011")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3012")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3013")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3014")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3015")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3016")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3017")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6007")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+        if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6018")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6019")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6020")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6021")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6022")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6023")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6024")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6025")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6026")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6027")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_call,
+                onpass="Re-assigned switch mastership to designated controller",
+                onfail="Something wrong with device_role calls")
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_check,
+                onpass="Switches were successfully reassigned to designated controller",
+                onfail="Switches were not successfully reassigned")
+        mastership_check = mastership_check and role_call and role_check
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
 
 
     def CASE3(self,main) :
@@ -283,6 +480,16 @@
         main.step("Discovering  Hosts( Via pingall for now)")
         #FIXME: Once we have a host discovery mechanism, use that instead
 
+        #install onos-app-fwd
+        main.log.info("Install reactive forwarding app")
+        main.ONOScli1.feature_install("onos-app-fwd")
+        main.ONOScli2.feature_install("onos-app-fwd")
+        main.ONOScli3.feature_install("onos-app-fwd")
+        main.ONOScli4.feature_install("onos-app-fwd")
+        main.ONOScli5.feature_install("onos-app-fwd")
+        main.ONOScli6.feature_install("onos-app-fwd")
+        main.ONOScli7.feature_install("onos-app-fwd")
+
         #REACTIVE FWD test
         ping_result = main.FALSE
         time1 = time.time()
@@ -299,22 +506,30 @@
         main.ONOScli5.feature_uninstall("onos-app-fwd")
         main.ONOScli6.feature_uninstall("onos-app-fwd")
         main.ONOScli7.feature_uninstall("onos-app-fwd")
+        #timeout for fwd flows
+        time.sleep(10)
 
         main.step("Add  host intents")
         #TODO:  move the host numbers to params
         import json
         intents_json= json.loads(main.ONOScli1.hosts())
-        intent_add_result = main.FALSE
+        intent_add_result = True
         for i in range(8,18):
             main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
             host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
             host2 =  "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
-            #NOTE: get host can return None
-            #TODO: handle this
             host1_id = main.ONOScli1.get_host(host1)['id']
             host2_id = main.ONOScli1.get_host(host2)['id']
-            tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
-            intent_add_result = intent_add_result and tmp_result
+            #NOTE: get host can return None
+            if host1_id and host2_id:
+                tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+            else:
+                main.log.error("Error, get_host() failed")
+                tmp_result = main.FALSE
+            intent_add_result = bool(intent_add_result and tmp_result)
+        utilities.assert_equals(expect=True, actual=intent_add_result,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
         #TODO Check if intents all exist in datastore
         #NOTE: Do we need to print this once the test is working?
         #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -360,6 +575,25 @@
 
         main.step("Get the Mastership of each switch from each controller")
         global mastership_state
+        mastership_state = []
+
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
         ONOS3_mastership = main.ONOScli3.roles()
@@ -416,6 +650,7 @@
 
         main.step("Get the intents from each controller")
         global intent_state
+        intent_state = []
         ONOS1_intents = main.ONOScli1.intents( json_format=True )
         ONOS2_intents = main.ONOScli2.intents( json_format=True )
         ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -470,6 +705,7 @@
 
         main.step("Get the flows from each controller")
         global flow_state
+        flow_state = []
         ONOS1_flows = main.ONOScli1.flows( json_format=True )
         ONOS2_flows = main.ONOScli2.flows( json_format=True )
         ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -527,7 +763,7 @@
         global flows
         flows=[]
         for i in range(1,29):
-            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+            flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
 
         #TODO: Compare switch flow tables with ONOS flow tables
 
@@ -578,16 +814,18 @@
         devices.append( main.ONOScli5.devices() )
         devices.append( main.ONOScli6.devices() )
         devices.append( main.ONOScli7.devices() )
-        '''
         hosts = []
-        hosts.append( main.ONOScli1.hosts() )
-        hosts.append( main.ONOScli2.hosts() )
-        hosts.append( main.ONOScli3.hosts() )
-        hosts.append( main.ONOScli4.hosts() )
-        hosts.append( main.ONOScli5.hosts() )
-        hosts.append( main.ONOScli6.hosts() )
-        hosts.append( main.ONOScli7.hosts() )
-        '''
+        hosts.append( json.loads( main.ONOScli1.hosts() ) )
+        hosts.append( json.loads( main.ONOScli2.hosts() ) )
+        hosts.append( json.loads( main.ONOScli3.hosts() ) )
+        hosts.append( json.loads( main.ONOScli4.hosts() ) )
+        hosts.append( json.loads( main.ONOScli5.hosts() ) )
+        hosts.append( json.loads( main.ONOScli6.hosts() ) )
+        hosts.append( json.loads( main.ONOScli7.hosts() ) )
+        for controller in range(0, len(hosts) ):
+            for host in hosts[controller]:
+                if host['ips'] == []:
+                    main.log.error("DEBUG:Error with host ips on controller"+str(controller+1)+": " + str(host))
         ports = []
         ports.append( main.ONOScli1.ports() )
         ports.append( main.ONOScli2.ports() )
@@ -604,13 +842,98 @@
         links.append( main.ONOScli5.links() )
         links.append( main.ONOScli6.links() )
         links.append( main.ONOScli7.links() )
+        clusters = []
+        clusters.append( main.ONOScli1.clusters() )
+        clusters.append( main.ONOScli2.clusters() )
+        clusters.append( main.ONOScli3.clusters() )
+        clusters.append( main.ONOScli4.clusters() )
+        clusters.append( main.ONOScli5.clusters() )
+        clusters.append( main.ONOScli6.clusters() )
+        clusters.append( main.ONOScli7.clusters() )
+        paths = []
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+        paths.append( temp_topo.get('paths', False) )
+
+        #Compare json objects for hosts, dataplane clusters and paths
+
+        #hosts
+        consistent_hosts_result = main.TRUE
+        for controller in range( len( hosts ) ):
+            if not "Error" in hosts[controller]:
+                if hosts[controller] == hosts[0]:
+                    continue
+                else:#hosts not consistent
+                    main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    main.log.warn( repr( hosts[controller] ) )
+                    consistent_hosts_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                consistent_hosts_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                onpass="Hosts view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of hosts")
+
+        #Strongly connected clusters of devices
+        consistent_clusters_result = main.TRUE
+        for controller in range( len( clusters ) ):
+            if not "Error" in clusters[controller]:
+                if clusters[controller] == clusters[0]:
+                    continue
+                else:#clusters not consistent
+                    main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_clusters_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                consistent_clusters_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                onpass="Clusters view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of clusters")
+        num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+        utilities.assert_equals(expect = 1, actual = num_clusters,
+                onpass="ONOS shows 1 SCC",
+                onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+        #paths
+        consistent_paths_result = main.TRUE
+        for controller in range( len( paths ) ):
+            if not "Error" in paths[controller]:
+                if paths[controller] == paths[0]:
+                    continue
+                else:#paths not consistent
+                    main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_paths_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                consistent_paths_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                onpass="Paths count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different counts of paths")
 
 
         main.step("Comparing ONOS topology to MN")
         devices_results = main.TRUE
         ports_results = main.TRUE
         links_results = main.TRUE
-        for controller in range(7): #TODO parameterize the number of controllers
+        for controller in range(num_controllers):
             if devices[controller] or not "Error" in devices[controller]:
                 current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
             else:
@@ -639,14 +962,16 @@
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
 
-        topo_result = devices_results and ports_results and links_results
+        topo_result = devices_results and ports_results and links_results\
+                and consistent_hosts_result and consistent_clusters_result\
+                and consistent_paths_result
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
                 onfail="Topology Check Test NOT successful")
 
         final_assert = main.TRUE
         final_assert = final_assert and topo_result and flow_check \
-                and intent_check and consistent_mastership
+                and intent_check and consistent_mastership and roles_not_null
         utilities.assert_equals(expect=main.TRUE, actual=final_assert,
                 onpass="State check successful",
                 onfail="State check NOT successful")
@@ -656,11 +981,14 @@
         '''
         The Failure case.
         '''
+        import time
         main.log.report("Killing 3 ONOS nodes")
         main.log.case("Restart minority of ONOS nodes")
         #TODO: Randomize these nodes
         main.ONOSbench.onos_kill(ONOS1_ip)
+        time.sleep(10)
         main.ONOSbench.onos_kill(ONOS2_ip)
+        time.sleep(10)
         main.ONOSbench.onos_kill(ONOS3_ip)
 
         main.step("Checking if ONOS is up yet")
@@ -680,6 +1008,8 @@
         cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
         cli_results = cli_result1 and cli_result2 and cli_result3
 
+        main.log.info("Install leadership election app on restarted node")
+
         case_results = main.TRUE and onos_isup_result and cli_results
         utilities.assert_equals(expect=main.TRUE, actual=case_results,
                 onpass="ONOS restart successful",
@@ -694,6 +1024,24 @@
         import json
         main.case("Running ONOS Constant State Tests")
 
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
+
         main.step("Check if switch roles are consistent across all nodes")
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
@@ -702,20 +1050,6 @@
         ONOS5_mastership = main.ONOScli5.roles()
         ONOS6_mastership = main.ONOScli6.roles()
         ONOS7_mastership = main.ONOScli7.roles()
-        print type(ONOS1_mastership)
-        print ONOS1_mastership
-        print type(ONOS2_mastership)
-        print ONOS2_mastership
-        print type(ONOS3_mastership)
-        print ONOS3_mastership
-        print type(ONOS4_mastership)
-        print ONOS4_mastership
-        print type(ONOS5_mastership)
-        print ONOS5_mastership
-        print type(ONOS6_mastership)
-        print ONOS6_mastership
-        print type(ONOS7_mastership)
-        print ONOS7_mastership
         #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
         if "Error" in ONOS1_mastership or not ONOS1_mastership\
                 or "Error" in ONOS2_mastership or not ONOS2_mastership\
@@ -739,7 +1073,6 @@
                 and ONOS1_mastership == ONOS5_mastership\
                 and ONOS1_mastership == ONOS6_mastership\
                 and ONOS1_mastership == ONOS7_mastership:
-                    #mastership_state = ONOS1_mastership
                     consistent_mastership = main.TRUE
                     main.log.report("Switch roles are consistent across all ONOS nodes")
         else:
@@ -785,7 +1118,7 @@
                 onpass="Mastership of Switches was not changed",
                 onfail="Mastership of some switches changed")
         #NOTE: we expect mastership to change on controller failure
-        mastership_check = mastership_check #and consistent_mastership
+        mastership_check = consistent_mastership
 
 
 
@@ -822,30 +1155,46 @@
                     intent_check = main.TRUE
                     main.log.report("Intents are consistent across all ONOS nodes")
         else:
-            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS1 intents: ")
+            print json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS2 intents: ")
+            print json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS3 intents: ")
+            print json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS4 intents: ")
+            print json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS5 intents: ")
+            print json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS6 intents: ")
+            print json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS7 intents: ")
+            print json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
         utilities.assert_equals(expect = main.TRUE,actual=intent_check,
                 onpass="Intents are consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of intents")
 
+        #NOTE: Hazelcast has no durability, so intents are lost across system restarts
         main.step("Compare current intents with intents before the failure")
+        #NOTE: this requires case 5 to pass for intent_state to be set.
+        #      maybe we should stop the test if that fails?
         if intent_state == ONOS1_intents:
             same_intents = main.TRUE
             main.log.report("Intents are consistent with before failure")
         #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
         else:
+            try:
+                main.log.warn("ONOS1 intents: ")
+                print json.dumps(json.loads(ONOS1_intents),
+                    sort_keys=True, indent=4, separators=(',', ': '))
+            except:
+                pass
             same_intents = main.FALSE
         utilities.assert_equals(expect = main.TRUE,actual=same_intents,
                 onpass="Intents are consistent with before failure",
@@ -859,12 +1208,12 @@
         flows2=[]
         for i in range(28):
             main.log.info("Checking flow table on s" + str(i+1))
-            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
             flows2.append(tmp_flows)
-            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            Flow_Tables = Flow_Tables and temp_result
             if Flow_Tables == main.FALSE:
                 main.log.info("Differences in flow table for switch: "+str(i+1))
-                break
         if Flow_Tables == main.TRUE:
             main.log.report("No changes were found in the flow tables")
         utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -880,7 +1229,7 @@
         #NOTE: checkForLoss returns main.FALSE with 0% packet loss
         for i in range(8,18):
             main.log.info("Checking for a loss in pings along flow from s" + str(i))
-            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+            Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
         if Loss_In_Pings == main.TRUE:
             main.log.info("Loss in ping detected")
         elif Loss_In_Pings == main.ERROR:
@@ -893,8 +1242,36 @@
                 onfail="Loss of dataplane connectivity detected")
 
 
-        #TODO:add topology to this or leave as a seperate case?
-        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        #Test of LeadershipElection
+        leader_list = []
+        leader_result = main.TRUE
+        for controller in range(1,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            leaderN = node.election_test_leader()
+            leader_list.append(leaderN)
+            if leaderN == main.FALSE:
+                #error in  response
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+            elif leaderN == None:
+                main.log.report("ONOS"+str(controller) + " shows no leader for the election-app was elected after the old one died")
+                leader_result = main.FALSE
+            elif leaderN == ONOS1_ip or leaderN == ONOS2_ip or leaderN == ONOS3_ip:
+                main.log.report("ONOS"+str(controller) + " shows "+str(leaderN)+" as leader for the election-app, but it was restarted")
+                leader_result = main.FALSE
+        if len( set( leader_list ) ) != 1:
+            leader_result = main.FALSE
+            main.log.error("Inconsistent view of leader for the election test app")
+            #TODO: print the list
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+                and leader_result
         result = int(result)
         if result == main.TRUE:
             main.log.report("Constant State Tests Passed")
@@ -936,87 +1313,180 @@
         ports_results = main.TRUE
         links_results = main.TRUE
         topo_result = main.FALSE
-        start_time = time.time()
         elapsed = 0
         count = 0
-        while topo_result == main.FALSE and elapsed < 120:
+        main.step("Collecting topology information from ONOS")
+        start_time = time.time()
+        while topo_result == main.FALSE and elapsed < 60:
             count = count + 1
-            try:
-                main.step("Collecting topology information from ONOS")
-                devices = []
-                devices.append( main.ONOScli1.devices() )
-                devices.append( main.ONOScli2.devices() )
-                devices.append( main.ONOScli3.devices() )
-                devices.append( main.ONOScli4.devices() )
-                devices.append( main.ONOScli5.devices() )
-                devices.append( main.ONOScli6.devices() )
-                devices.append( main.ONOScli7.devices() )
-                '''
-                hosts = []
-                hosts.append( main.ONOScli1.hosts() )
-                hosts.append( main.ONOScli2.hosts() )
-                hosts.append( main.ONOScli3.hosts() )
-                hosts.append( main.ONOScli4.hosts() )
-                hosts.append( main.ONOScli5.hosts() )
-                hosts.append( main.ONOScli6.hosts() )
-                hosts.append( main.ONOScli7.hosts() )
-                '''
-                ports = []
-                ports.append( main.ONOScli1.ports() )
-                ports.append( main.ONOScli2.ports() )
-                ports.append( main.ONOScli3.ports() )
-                ports.append( main.ONOScli4.ports() )
-                ports.append( main.ONOScli5.ports() )
-                ports.append( main.ONOScli6.ports() )
-                ports.append( main.ONOScli7.ports() )
-                links = []
-                links.append( main.ONOScli1.links() )
-                links.append( main.ONOScli2.links() )
-                links.append( main.ONOScli3.links() )
-                links.append( main.ONOScli4.links() )
-                links.append( main.ONOScli5.links() )
-                links.append( main.ONOScli6.links() )
-                links.append( main.ONOScli7.links() )
+            if count > 1:
+                MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+            cli_start = time.time()
+            devices = []
+            devices.append( main.ONOScli1.devices() )
+            devices.append( main.ONOScli2.devices() )
+            devices.append( main.ONOScli3.devices() )
+            devices.append( main.ONOScli4.devices() )
+            devices.append( main.ONOScli5.devices() )
+            devices.append( main.ONOScli6.devices() )
+            devices.append( main.ONOScli7.devices() )
+            hosts = []
+            hosts.append( json.loads( main.ONOScli1.hosts() ) )
+            hosts.append( json.loads( main.ONOScli2.hosts() ) )
+            hosts.append( json.loads( main.ONOScli3.hosts() ) )
+            hosts.append( json.loads( main.ONOScli4.hosts() ) )
+            hosts.append( json.loads( main.ONOScli5.hosts() ) )
+            hosts.append( json.loads( main.ONOScli6.hosts() ) )
+            hosts.append( json.loads( main.ONOScli7.hosts() ) )
+            for controller in range(0, len(hosts) ):
+                for host in hosts[controller]:
+                    host
+                    if host['ips'] == []:
+                        main.log.error("DEBUG:Error with host ips on controller"+str(controller+1)+": " + str(host))
+            ports = []
+            ports.append( main.ONOScli1.ports() )
+            ports.append( main.ONOScli2.ports() )
+            ports.append( main.ONOScli3.ports() )
+            ports.append( main.ONOScli4.ports() )
+            ports.append( main.ONOScli5.ports() )
+            ports.append( main.ONOScli6.ports() )
+            ports.append( main.ONOScli7.ports() )
+            links = []
+            links.append( main.ONOScli1.links() )
+            links.append( main.ONOScli2.links() )
+            links.append( main.ONOScli3.links() )
+            links.append( main.ONOScli4.links() )
+            links.append( main.ONOScli5.links() )
+            links.append( main.ONOScli6.links() )
+            links.append( main.ONOScli7.links() )
+            clusters = []
+            clusters.append( main.ONOScli1.clusters() )
+            clusters.append( main.ONOScli2.clusters() )
+            clusters.append( main.ONOScli3.clusters() )
+            clusters.append( main.ONOScli4.clusters() )
+            clusters.append( main.ONOScli5.clusters() )
+            clusters.append( main.ONOScli6.clusters() )
+            clusters.append( main.ONOScli7.clusters() )
+            paths = []
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+            paths.append( temp_topo.get('paths', False) )
 
-                for controller in range(7): #TODO parameterize the number of controllers
-                    if devices[controller] or not "Error" in devices[controller]:
-                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
-                    else:
-                        current_devices_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
-                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
-                    if ports[controller] or not "Error" in ports[controller]:
-                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
-                    else:
-                        current_ports_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
-                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+            elapsed = time.time() - start_time
+            cli_time = time.time() - cli_start
+            print "CLI time: " + str(cli_time)
 
-                    if links[controller] or not "Error" in links[controller]:
-                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
-                    else:
-                        current_links_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
-                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
-            except:
-                main.log.error("something went wrong in topo comparison")
-                main.log.warn( repr( devices ) )
-                main.log.warn( repr( ports ) )
-                main.log.warn( repr( links ) )
+            for controller in range(num_controllers):
+                if devices[controller] or not "Error" in devices[controller]:
+                    current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                else:
+                    current_devices_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                        onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
+                if ports[controller] or not "Error" in ports[controller]:
+                    current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                else:
+                    current_ports_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                        onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                if links[controller] or not "Error" in links[controller]:
+                    current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                else:
+                    current_links_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                        onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
             devices_results = devices_results and current_devices_result
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
-            topo_result = devices_results and ports_results and links_results
-            elapsed = time.time() - start_time
-        time_threshold = elapsed < 1
-        topo_result = topo_result and time_threshold
-        #TODO make sure this step is non-blocking. IE add a timeout
-        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+            #Compare json objects for hosts, dataplane clusters and paths
+
+            #hosts
+            consistent_hosts_result = main.TRUE
+            for controller in range( len( hosts ) ):
+                if not "Error" in hosts[controller]:
+                    if hosts[controller] == hosts[0]:
+                        continue
+                    else:#hosts not consistent
+                        main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        main.log.warn( repr( hosts[controller] ) )
+                        consistent_hosts_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                    consistent_hosts_result = main.FALSE
+            if consistent_hosts_result == main.FALSE:
+                for controller in range( len( hosts ) ):
+                    main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                    onpass="Hosts view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of hosts")
+
+            #Strongly connected clusters of devices
+            consistent_clusters_result = main.TRUE
+            for controller in range( len( clusters ) ):
+                if not "Error" in clusters[controller]:
+                    if clusters[controller] == clusters[0]:
+                        continue
+                    else:#clusters not consistent
+                        main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_clusters_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                    consistent_clusters_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                    onpass="Clusters view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of clusters")
+            num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+            utilities.assert_equals(expect = 1, actual = num_clusters,
+                    onpass="ONOS shows 1 SCC",
+                    onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+            #paths
+            consistent_paths_result = main.TRUE
+            for controller in range( len( paths ) ):
+                if not "Error" in paths[controller]:
+                    if paths[controller] == paths[0]:
+                        continue
+                    else:#paths not consistent
+                        main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_paths_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                    consistent_paths_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                    onpass="Paths count is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different counts of paths")
+
+
+            topo_result = devices_results and ports_results and links_results\
+                    and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+        topo_result = topo_result and int(count <= 2)
+        note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+        main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
                 str(elapsed) + " seconds, " + str(count) +" tries" )
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
@@ -1031,7 +1501,7 @@
         '''
         #NOTE: You should probably run a topology check after this
 
-        link_sleep = int(main.params['timers']['LinkDiscovery'])
+        link_sleep = float(main.params['timers']['LinkDiscovery'])
 
         description = "Turn off a link to ensure that Link Discovery is working properly"
         main.log.report(description)
@@ -1053,7 +1523,7 @@
         '''
         #NOTE: You should probably run a topology check after this
 
-        link_sleep = int(main.params['timers']['LinkDiscovery'])
+        link_sleep = float(main.params['timers']['LinkDiscovery'])
 
         description = "Restore a link to ensure that Link Discovery is working properly"
         main.log.report(description)
@@ -1076,7 +1546,7 @@
         #NOTE: You should probably run a topology check after this
         import time
 
-        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+        switch_sleep = float(main.params['timers']['SwitchDiscovery'])
 
         description = "Killing a switch to ensure it is discovered correctly"
         main.log.report(description)
@@ -1085,13 +1555,18 @@
         #TODO: Make this switch parameterizable
         main.step("Kill s28 ")
         main.log.report("Deleting s28")
-        #FIXME: use new dynamic topo functions
         main.Mininet1.del_switch("s28")
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
         time.sleep(switch_sleep)
+        device = main.ONOScli1.get_device(dpid="0028")
         #Peek at the deleted switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == False:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Kill switch succesful",
+                onfail="Failed to kill switch?")
 
     def CASE12 (self, main) :
         '''
@@ -1099,7 +1574,8 @@
         '''
         #NOTE: You should probably run a topology check after this
         import time
-        #FIXME: use new dynamic topo functions
+
+        switch_sleep = float(main.params['timers']['SwitchDiscovery'])
         description = "Adding a switch to ensure it is discovered correctly"
         main.log.report(description)
         main.case(description)
@@ -1111,7 +1587,7 @@
         main.Mininet1.add_link('s28', 's3')
         main.Mininet1.add_link('s28', 's6')
         main.Mininet1.add_link('s28', 'h28')
-        main.Mininet1.assign_sw_controller(sw="28",count=7,
+        main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
                 ip1=ONOS1_ip,port1=ONOS1_port,
                 ip2=ONOS2_ip,port2=ONOS2_port,
                 ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1121,9 +1597,15 @@
                 ip7=ONOS7_ip,port7=ONOS7_port)
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
         time.sleep(switch_sleep)
-        #Peek at the added switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        device = main.ONOScli1.get_device(dpid="0028")
+        #Peek at the deleted switch
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == True:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="add switch succesful",
+                onfail="Failed to add switch?")
 
     def CASE13 (self, main) :
         '''
@@ -1131,14 +1613,41 @@
         '''
         import os
         import time
+        #printing colors to terminal
+        colors = {}
+        colors['cyan']   = '\033[96m'
+        colors['purple'] = '\033[95m'
+        colors['blue']   = '\033[94m'
+        colors['green']  = '\033[92m'
+        colors['yellow'] = '\033[93m'
+        colors['red']    = '\033[91m'
+        colors['end']    = '\033[0m'
         description = "Test Cleanup"
         main.log.report(description)
         main.case(description)
         main.step("Killing tcpdumps")
         main.Mininet2.stop_tcpdump()
 
+        main.step("Checking ONOS Logs for errors")
+        print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS1_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS2_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS3_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS4_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS5_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS6_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS7_ip)
+
         main.step("Copying MN pcap and ONOS log files to test station")
         testname = main.TEST
+        teststation_user = main.params['TESTONUSER']
+        teststation_IP = main.params['TESTONIP']
         #NOTE: MN Pcap file is being saved to ~/packet_captures
         #       scp this file as MN and TestON aren't necessarily the same vm
         #FIXME: scp
@@ -1150,19 +1659,26 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
         #std*.log's
@@ -1172,24 +1688,29 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
 
-
-
         #sleep so scp can finish
         time.sleep(10)
         main.step("Packing and rotating pcap archives")
@@ -1200,3 +1721,139 @@
         utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
                 onpass="Test cleanup successful",
                 onfail="Test cleanup NOT successful")
+
+    def CASE14 ( self, main ) :
+        '''
+        start election app on all onos nodes
+        '''
+        leader_result = main.TRUE
+        #install app on onos 1
+        main.log.info("Install leadership election app")
+        main.ONOScli1.feature_install("onos-app-election")
+        #wait for election
+        #check for leader
+        leader = main.ONOScli1.election_test_leader()
+        #verify leader is ONOS1
+        if leader == ONOS1_ip:
+            #all is well
+            pass
+        elif leader == None:
+            #No leader elected
+            main.log.report("No leader was elected")
+            leader_result = main.FALSE
+        elif leader == main.FALSE:
+            #error in  response
+            #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+            main.log.report("Something is wrong with election_test_leader function, check the error logs")
+            leader_result = main.FALSE
+        else:
+            #error in  response
+            main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+            leader_result = main.FALSE
+
+
+
+
+        #install on other nodes and check for leader.
+        #Should be onos1 and each app should show the same leader
+        for controller in range(2,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            node.feature_install("onos-app-election")
+            leaderN = node.election_test_leader()
+            #verify leader is ONOS1
+            if leaderN == ONOS1_ip:
+                #all is well
+                pass
+            elif leaderN == main.FALSE:
+                #error in  response
+                #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+            elif leader != leaderN:
+                leader_result = main.FALSE
+                main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+                        " as the leader of the election app. Leader should be "+str(leader) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+    def CASE15 ( self, main ) :
+        '''
+        Check that Leadership Election is still functional
+        '''
+        leader_result = main.TRUE
+        description = "Check that Leadership Election is still functional"
+        main.log.report(description)
+        main.case(description)
+        main.step("Find current leader and withdraw")
+        leader = main.ONOScli1.election_test_leader()
+        #TODO: do some sanity checking on leader before using it
+        withdraw_result = main.FALSE
+        if leader == ONOS1_ip:
+            old_leader = getattr( main, "ONOScli1" )
+        elif leader == ONOS2_ip:
+            old_leader = getattr( main, "ONOScli2" )
+        elif leader == ONOS3_ip:
+            old_leader = getattr( main, "ONOScli3" )
+        elif leader == ONOS4_ip:
+            old_leader = getattr( main, "ONOScli4" )
+        elif leader == ONOS5_ip:
+            old_leader = getattr( main, "ONOScli5" )
+        elif leader == ONOS6_ip:
+            old_leader = getattr( main, "ONOScli6" )
+        elif leader == ONOS7_ip:
+            old_leader = getattr( main, "ONOScli7" )
+        elif leader == None or leader == main.FALSE:
+            main.log.report("Leader for the election app should be an ONOS node,"\
+                    +"instead got '"+str(leader)+"'")
+            leader_result = main.FALSE
+        withdraw_result = old_leader.election_test_withdraw()
+
+
+        main.step("Make sure new leader is elected")
+        leader_list = []
+        for controller in range(1,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            leader_list.append( node.election_test_leader() )
+        for leaderN in leader_list:
+            if leaderN == leader:
+                main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+                        " as leader after they withdrew")
+                leader_result = main.FALSE
+            elif leaderN == main.FALSE:
+                #error in  response
+                #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+        consistent_leader = main.FALSE
+        if len( set( leader_list ) ) == 1:
+            main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+            consistent_leader = main.TRUE
+        else:
+            main.log.report("Inconsistent responses for leader of Election-app:")
+            for n in range(len(leader_list)):
+                main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+
+        main.step("Run for election on old leader(just so everyone is in the hat)")
+        run_result = old_leader.election_test_run()
+        if consistent_leader == main.TRUE:
+            after_run = main.ONOScli1.election_test_leader()
+            #verify leader didn't just change
+            if after_run == leader_list[0]:
+                leader_result = main.TRUE
+            else:
+                leader_result = main.FALSE
+        #TODO: assert on  run and withdraw results?
+
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index e1e75f9..78e0c83 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,9 +1,10 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
     <cellName>HA</cellName>
     </ENV>
-    <Git>True</Git>
+    <Git>False</Git>
+    <num_controllers> 7 </num_controllers>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
@@ -28,7 +29,7 @@
         <port7>6633</port7>
     </CTRL>
     <TESTONUSER>admin</TESTONUSER>
-    <TESTONIP>10.128.30.10</TESTONIP>
+    <TESTONIP>10.128.30.9</TESTONIP>
     <PING>
         <source1>h8</source1>
         <source2>h9</source2>
@@ -52,8 +53,8 @@
         <target10>10.0.0.27</target10>
     </PING>
     <timers>
-        <LinkDiscovery>2</LinkDiscovery>
-        <SwitchDiscovery>2</SwitchDiscovery>
+        <LinkDiscovery>.2</LinkDiscovery>
+        <SwitchDiscovery>.2</SwitchDiscovery>
     </timers>
     <MNtcpdump>
         <intf>eth0</intf>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 614d514..7399925 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -17,6 +17,8 @@
 CASE11: Switch down
 CASE12: Switch up
 CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
 '''
 class HATestSanity:
 
@@ -63,6 +65,7 @@
         global ONOS6_port
         global ONOS7_ip
         global ONOS7_port
+        global num_controllers
 
         ONOS1_ip = main.params['CTRL']['ip1']
         ONOS1_port = main.params['CTRL']['port1']
@@ -78,6 +81,7 @@
         ONOS6_port = main.params['CTRL']['port6']
         ONOS7_ip = main.params['CTRL']['ip7']
         ONOS7_port = main.params['CTRL']['port7']
+        num_controllers = int(main.params['num_controllers'])
 
 
         main.step("Applying cell variable to environment")
@@ -145,8 +149,7 @@
 
 
         main.step("Checking if ONOS is up yet")
-        #TODO: Refactor
-        # check bundle:list?
+        #TODO check bundle:list?
         for i in range(2):
             onos1_isup = main.ONOSbench.isup(ONOS1_ip)
             if not onos1_isup:
@@ -173,7 +176,6 @@
                     and onos4_isup and onos5_isup and onos6_isup and onos7_isup
             if onos_isup_result == main.TRUE:
                 break
-        # TODO: if it becomes an issue, we can retry this step  a few times
 
 
         cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
@@ -214,31 +216,12 @@
         import json
         import re
 
-
-        '''
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS1_port = main.params['CTRL']['port1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS2_port = main.params['CTRL']['port2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS3_port = main.params['CTRL']['port3']
-        ONOS4_ip = main.params['CTRL']['ip4']
-        ONOS4_port = main.params['CTRL']['port4']
-        ONOS5_ip = main.params['CTRL']['ip5']
-        ONOS5_port = main.params['CTRL']['port5']
-        ONOS6_ip = main.params['CTRL']['ip6']
-        ONOS6_port = main.params['CTRL']['port6']
-        ONOS7_ip = main.params['CTRL']['ip7']
-        ONOS7_port = main.params['CTRL']['port7']
-        '''
-
-
         main.log.report("Assigning switches to controllers")
         main.case("Assigning Controllers")
         main.step("Assign switches to controllers")
 
         for i in range (1,29):
-           main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+           main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
                     ip1=ONOS1_ip,port1=ONOS1_port,
                     ip2=ONOS2_ip,port2=ONOS2_port,
                     ip3=ONOS3_ip,port3=ONOS3_port,
@@ -270,8 +253,217 @@
                 onpass="Switch mastership assigned correctly",
                 onfail="Switches not assigned correctly to controllers")
 
-        #TODO: If assign roles is working reliably then manually 
-        #   assign mastership to the controller we want
+        #Manually assign mastership to the controller we want
+        role_call = main.TRUE
+        role_check = main.TRUE
+
+        device_id = main.ONOScli1.get_device("1000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2800")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+        if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("2000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+        if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("5000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6000")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+        if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id =  main.ONOScli1.get_device("3004")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+        if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3008")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3009")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3010")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3011")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3012")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3013")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3014")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3015")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3016")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("3017")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+        if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6007")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+        if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6018")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6019")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6020")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6021")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6022")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6023")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6024")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6025")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6026")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        device_id = main.ONOScli1.get_device("6027")['id']
+        role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+        if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+            role_check = role_check and main.TRUE
+        else:
+            role_check = role_check and main.FALSE
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_call,
+                onpass="Re-assigned switch mastership to designated controller",
+                onfail="Something wrong with device_role calls")
+
+        utilities.assert_equals(expect = main.TRUE,actual=role_check,
+                onpass="Switches were successfully reassigned to designated controller",
+                onfail="Switches were not successfully reassigned")
+        mastership_check = mastership_check and role_call and role_check
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
 
 
     def CASE3(self,main) :
@@ -288,6 +480,16 @@
         main.step("Discovering  Hosts( Via pingall for now)")
         #FIXME: Once we have a host discovery mechanism, use that instead
 
+        #install onos-app-fwd
+        main.log.info("Install reactive forwarding app")
+        main.ONOScli1.feature_install("onos-app-fwd")
+        main.ONOScli2.feature_install("onos-app-fwd")
+        main.ONOScli3.feature_install("onos-app-fwd")
+        main.ONOScli4.feature_install("onos-app-fwd")
+        main.ONOScli5.feature_install("onos-app-fwd")
+        main.ONOScli6.feature_install("onos-app-fwd")
+        main.ONOScli7.feature_install("onos-app-fwd")
+
         #REACTIVE FWD test
         ping_result = main.FALSE
         time1 = time.time()
@@ -304,22 +506,30 @@
         main.ONOScli5.feature_uninstall("onos-app-fwd")
         main.ONOScli6.feature_uninstall("onos-app-fwd")
         main.ONOScli7.feature_uninstall("onos-app-fwd")
+        #timeout for fwd flows
+        time.sleep(10)
 
         main.step("Add  host intents")
         #TODO:  move the host numbers to params
         import json
         intents_json= json.loads(main.ONOScli1.hosts())
-        intent_add_result = main.FALSE
+        intent_add_result = True
         for i in range(8,18):
             main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
             host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
             host2 =  "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
-            #NOTE: get host can return None
-            #TODO: handle this
             host1_id = main.ONOScli1.get_host(host1)['id']
             host2_id = main.ONOScli1.get_host(host2)['id']
-            tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
-            intent_add_result = intent_add_result and tmp_result
+            #NOTE: get host can return None
+            if host1_id and host2_id:
+                tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+            else:
+                main.log.error("Error, get_host() failed")
+                tmp_result = main.FALSE
+            intent_add_result = bool(intent_add_result and tmp_result)
+        utilities.assert_equals(expect=True, actual=intent_add_result,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
         #TODO Check if intents all exist in datastore
         #NOTE: Do we need to print this once the test is working?
         #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -365,6 +575,25 @@
 
         main.step("Get the Mastership of each switch from each controller")
         global mastership_state
+        mastership_state = []
+
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
         ONOS3_mastership = main.ONOScli3.roles()
@@ -421,6 +650,7 @@
 
         main.step("Get the intents from each controller")
         global intent_state
+        intent_state = []
         ONOS1_intents = main.ONOScli1.intents( json_format=True )
         ONOS2_intents = main.ONOScli2.intents( json_format=True )
         ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -475,6 +705,7 @@
 
         main.step("Get the flows from each controller")
         global flow_state
+        flow_state = []
         ONOS1_flows = main.ONOScli1.flows( json_format=True )
         ONOS2_flows = main.ONOScli2.flows( json_format=True )
         ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -532,7 +763,7 @@
         global flows
         flows=[]
         for i in range(1,29):
-            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+            flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
 
         #TODO: Compare switch flow tables with ONOS flow tables
 
@@ -583,7 +814,6 @@
         devices.append( main.ONOScli5.devices() )
         devices.append( main.ONOScli6.devices() )
         devices.append( main.ONOScli7.devices() )
-        '''
         hosts = []
         hosts.append( main.ONOScli1.hosts() )
         hosts.append( main.ONOScli2.hosts() )
@@ -592,7 +822,6 @@
         hosts.append( main.ONOScli5.hosts() )
         hosts.append( main.ONOScli6.hosts() )
         hosts.append( main.ONOScli7.hosts() )
-        '''
         ports = []
         ports.append( main.ONOScli1.ports() )
         ports.append( main.ONOScli2.ports() )
@@ -609,13 +838,98 @@
         links.append( main.ONOScli5.links() )
         links.append( main.ONOScli6.links() )
         links.append( main.ONOScli7.links() )
+        clusters = []
+        clusters.append( main.ONOScli1.clusters() )
+        clusters.append( main.ONOScli2.clusters() )
+        clusters.append( main.ONOScli3.clusters() )
+        clusters.append( main.ONOScli4.clusters() )
+        clusters.append( main.ONOScli5.clusters() )
+        clusters.append( main.ONOScli6.clusters() )
+        clusters.append( main.ONOScli7.clusters() )
+        paths = []
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+        paths.append( temp_topo.get('paths', False) )
+        temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+        paths.append( temp_topo.get('paths', False) )
+
+        #Compare json objects for hosts, dataplane clusters and paths
+
+        #hosts
+        consistent_hosts_result = main.TRUE
+        for controller in range( len( hosts ) ):
+            if not "Error" in hosts[controller]:
+                if hosts[controller] == hosts[0]:
+                    continue
+                else:#hosts not consistent
+                    main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    main.log.warn( repr( hosts[controller] ) )
+                    consistent_hosts_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                consistent_hosts_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                onpass="Hosts view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of hosts")
+
+        #Strongly connected clusters of devices
+        consistent_clusters_result = main.TRUE
+        for controller in range( len( clusters ) ):
+            if not "Error" in clusters[controller]:
+                if clusters[controller] == clusters[0]:
+                    continue
+                else:#clusters not consistent
+                    main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_clusters_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                consistent_clusters_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                onpass="Clusters view is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of clusters")
+        num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+        utilities.assert_equals(expect = 1, actual = num_clusters,
+                onpass="ONOS shows 1 SCC",
+                onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+        #paths
+        consistent_paths_result = main.TRUE
+        for controller in range( len( paths ) ):
+            if not "Error" in paths[controller]:
+                if paths[controller] == paths[0]:
+                    continue
+                else:#paths not consistent
+                    main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                    consistent_paths_result = main.FALSE
+
+            else:
+                main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                consistent_paths_result = main.FALSE
+                main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                onpass="Paths count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different counts of paths")
 
 
         main.step("Comparing ONOS topology to MN")
         devices_results = main.TRUE
         ports_results = main.TRUE
         links_results = main.TRUE
-        for controller in range(7): #TODO parameterize the number of controllers
+        for controller in range(num_controllers):
             if devices[controller] or not "Error" in devices[controller]:
                 current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
             else:
@@ -644,14 +958,16 @@
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
 
-        topo_result = devices_results and ports_results and links_results
+        topo_result = devices_results and ports_results and links_results\
+                and consistent_hosts_result and consistent_clusters_result\
+                and consistent_paths_result
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
                 onfail="Topology Check Test NOT successful")
 
         final_assert = main.TRUE
         final_assert = final_assert and topo_result and flow_check \
-                and intent_check and consistent_mastership
+                and intent_check and consistent_mastership and roles_not_null
         utilities.assert_equals(expect=main.TRUE, actual=final_assert,
                 onpass="State check successful",
                 onfail="State check NOT successful")
@@ -676,6 +992,24 @@
         import json
         main.case("Running ONOS Constant State Tests")
 
+        #Assert that each device has a master
+        ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+        ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+        ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+        ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+        ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+        ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+        ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+        roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+                ONOS3_master_not_null and ONOS4_master_not_null and\
+                ONOS5_master_not_null and ONOS6_master_not_null and\
+                ONOS7_master_not_null
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
+
         main.step("Check if switch roles are consistent across all nodes")
         ONOS1_mastership = main.ONOScli1.roles()
         ONOS2_mastership = main.ONOScli2.roles()
@@ -707,7 +1041,6 @@
                 and ONOS1_mastership == ONOS5_mastership\
                 and ONOS1_mastership == ONOS6_mastership\
                 and ONOS1_mastership == ONOS7_mastership:
-                    #mastership_state = ONOS1_mastership
                     consistent_mastership = main.TRUE
                     main.log.report("Switch roles are consistent across all ONOS nodes")
         else:
@@ -789,25 +1122,35 @@
                     intent_check = main.TRUE
                     main.log.report("Intents are consistent across all ONOS nodes")
         else:
-            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
-            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
-                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS1 intents: ") 
+            print json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS2 intents: ")
+            print json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS3 intents: ")
+            print json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS4 intents: ")
+            print json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS5 intents: ")
+            print json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS6 intents: ")
+            print json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
+            main.log.warn("ONOS7 intents: ")
+            print json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': '))
         utilities.assert_equals(expect = main.TRUE,actual=intent_check,
                 onpass="Intents are consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of intents")
 
+        #NOTE: Hazelcast has no durability, so intents are lost
         main.step("Compare current intents with intents before the failure")
+        #NOTE: this requires case 5 to pass for intent_state to be set.
+        #      maybe we should stop the test if that fails?
         if intent_state == ONOS1_intents:
             same_intents = main.TRUE
             main.log.report("Intents are consistent with before failure")
@@ -826,12 +1169,12 @@
         flows2=[]
         for i in range(28):
             main.log.info("Checking flow table on s" + str(i+1))
-            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
             flows2.append(tmp_flows)
-            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            Flow_Tables = Flow_Tables and temp_result
             if Flow_Tables == main.FALSE:
                 main.log.info("Differences in flow table for switch: "+str(i+1))
-                break
         if Flow_Tables == main.TRUE:
             main.log.report("No changes were found in the flow tables")
         utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -847,7 +1190,7 @@
         #NOTE: checkForLoss returns main.FALSE with 0% packet loss
         for i in range(8,18):
             main.log.info("Checking for a loss in pings along flow from s" + str(i))
-            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+            Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
         if Loss_In_Pings == main.TRUE:
             main.log.info("Loss in ping detected")
         elif Loss_In_Pings == main.ERROR:
@@ -859,9 +1202,35 @@
                 onpass="No Loss of connectivity",
                 onfail="Loss of dataplane connectivity detected")
 
+        #Test of LeadershipElection
+        #NOTE: this only works for the sanity test. In case of failures, leader will likely change
+        leader = ONOS1_ip
+        leader_result = main.TRUE
+        for controller in range(1,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            leaderN = node.election_test_leader()
+            #verify leader is ONOS1
+            if leaderN == leader:
+                #all is well
+                #NOTE: In failure scenario, this could be a new node, maybe check != ONOS1
+                pass
+            elif leaderN == main.FALSE:
+                #error in  response
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+            elif leader != leaderN:
+                leader_result = main.FALSE
+                main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+                        " as the leader of the election app. Leader should be "+str(leader) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
 
-        #TODO:add topology to this or leave as a seperate case?
-        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+                and leader_result
         result = int(result)
         if result == main.TRUE:
             main.log.report("Constant State Tests Passed")
@@ -903,89 +1272,173 @@
         ports_results = main.TRUE
         links_results = main.TRUE
         topo_result = main.FALSE
-        start_time = time.time()
         elapsed = 0
         count = 0
-        while topo_result == main.FALSE and elapsed < 120:
-            print "cond 1:" + str(topo_result == main.FALSE)
-            print "cond 2:" + str(elapsed < 120)
+        main.step("Collecting topology information from ONOS")
+        start_time = time.time()
+        while topo_result == main.FALSE and elapsed < 60:
             count = count + 1
-            try:
-                main.step("Collecting topology information from ONOS")
-                devices = []
-                devices.append( main.ONOScli1.devices() )
-                devices.append( main.ONOScli2.devices() )
-                devices.append( main.ONOScli3.devices() )
-                devices.append( main.ONOScli4.devices() )
-                devices.append( main.ONOScli5.devices() )
-                devices.append( main.ONOScli6.devices() )
-                devices.append( main.ONOScli7.devices() )
-                '''
-                hosts = []
-                hosts.append( main.ONOScli1.hosts() )
-                hosts.append( main.ONOScli2.hosts() )
-                hosts.append( main.ONOScli3.hosts() )
-                hosts.append( main.ONOScli4.hosts() )
-                hosts.append( main.ONOScli5.hosts() )
-                hosts.append( main.ONOScli6.hosts() )
-                hosts.append( main.ONOScli7.hosts() )
-                '''
-                ports = []
-                ports.append( main.ONOScli1.ports() )
-                ports.append( main.ONOScli2.ports() )
-                ports.append( main.ONOScli3.ports() )
-                ports.append( main.ONOScli4.ports() )
-                ports.append( main.ONOScli5.ports() )
-                ports.append( main.ONOScli6.ports() )
-                ports.append( main.ONOScli7.ports() )
-                links = []
-                links.append( main.ONOScli1.links() )
-                links.append( main.ONOScli2.links() )
-                links.append( main.ONOScli3.links() )
-                links.append( main.ONOScli4.links() )
-                links.append( main.ONOScli5.links() )
-                links.append( main.ONOScli6.links() )
-                links.append( main.ONOScli7.links() )
+            if count > 1:
+                MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+            cli_start = time.time()
+            devices = []
+            devices.append( main.ONOScli1.devices() )
+            devices.append( main.ONOScli2.devices() )
+            devices.append( main.ONOScli3.devices() )
+            devices.append( main.ONOScli4.devices() )
+            devices.append( main.ONOScli5.devices() )
+            devices.append( main.ONOScli6.devices() )
+            devices.append( main.ONOScli7.devices() )
+            hosts = []
+            hosts.append( main.ONOScli1.hosts() )
+            hosts.append( main.ONOScli2.hosts() )
+            hosts.append( main.ONOScli3.hosts() )
+            hosts.append( main.ONOScli4.hosts() )
+            hosts.append( main.ONOScli5.hosts() )
+            hosts.append( main.ONOScli6.hosts() )
+            hosts.append( main.ONOScli7.hosts() )
+            ports = []
+            ports.append( main.ONOScli1.ports() )
+            ports.append( main.ONOScli2.ports() )
+            ports.append( main.ONOScli3.ports() )
+            ports.append( main.ONOScli4.ports() )
+            ports.append( main.ONOScli5.ports() )
+            ports.append( main.ONOScli6.ports() )
+            ports.append( main.ONOScli7.ports() )
+            links = []
+            links.append( main.ONOScli1.links() )
+            links.append( main.ONOScli2.links() )
+            links.append( main.ONOScli3.links() )
+            links.append( main.ONOScli4.links() )
+            links.append( main.ONOScli5.links() )
+            links.append( main.ONOScli6.links() )
+            links.append( main.ONOScli7.links() )
+            clusters = []
+            clusters.append( main.ONOScli1.clusters() )
+            clusters.append( main.ONOScli2.clusters() )
+            clusters.append( main.ONOScli3.clusters() )
+            clusters.append( main.ONOScli4.clusters() )
+            clusters.append( main.ONOScli5.clusters() )
+            clusters.append( main.ONOScli6.clusters() )
+            clusters.append( main.ONOScli7.clusters() )
+            paths = []
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+            paths.append( temp_topo.get('paths', False) )
+            temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+            paths.append( temp_topo.get('paths', False) )
 
-                for controller in range(7): #TODO parameterize the number of controllers
-                    if devices[controller] or not "Error" in devices[controller]:
-                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
-                    else:
-                        current_devices_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
-                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
-                    if ports[controller] or not "Error" in ports[controller]:
-                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
-                    else:
-                        current_ports_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
-                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+            elapsed = time.time() - start_time
+            cli_time = time.time() - cli_start
+            print "CLI time: " + str(cli_time)
 
-                    if links[controller] or not "Error" in links[controller]:
-                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
-                    else:
-                        current_links_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
-                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
-            except:
-                main.log.error("something went wrong in topo comparison")
-                main.log.warn( repr( devices ) )
-                main.log.warn( repr( ports ) )
-                main.log.warn( repr( links ) )
+            for controller in range(num_controllers):
+                if devices[controller] or not "Error" in devices[controller]:
+                    current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                else:
+                    current_devices_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                        onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
+                if ports[controller] or not "Error" in ports[controller]:
+                    current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                else:
+                    current_ports_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                        onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                if links[controller] or not "Error" in links[controller]:
+                    current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                else:
+                    current_links_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                        onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
             devices_results = devices_results and current_devices_result
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
-            topo_result = devices_results and ports_results and links_results
-            elapsed = time.time() - start_time
-        time_threshold = elapsed < 1
-        topo_result = topo_result and time_threshold
-        #TODO make sure this step is non-blocking. IE add a timeout
-        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+            #Compare json objects for hosts, dataplane clusters and paths
+
+            #hosts
+            consistent_hosts_result = main.TRUE
+            for controller in range( len( hosts ) ):
+                if not "Error" in hosts[controller]:
+                    if hosts[controller] == hosts[0]:
+                        continue
+                    else:#hosts not consistent
+                        main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        main.log.warn( repr( hosts[controller] ) )
+                        consistent_hosts_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+                    consistent_hosts_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+                    onpass="Hosts view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of hosts")
+
+            #Strongly connected clusters of devices
+            consistent_clusters_result = main.TRUE
+            for controller in range( len( clusters ) ):
+                if not "Error" in clusters[controller]:
+                    if clusters[controller] == clusters[0]:
+                        continue
+                    else:#clusters not consistent
+                        main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_clusters_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+                    consistent_clusters_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+                    onpass="Clusters view is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different views of clusters")
+            num_clusters =  len(json.loads(clusters[0])) #there should always only be one cluster
+            utilities.assert_equals(expect = 1, actual = num_clusters,
+                    onpass="ONOS shows 1 SCC",
+                    onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+            #paths
+            consistent_paths_result = main.TRUE
+            for controller in range( len( paths ) ):
+                if not "Error" in paths[controller]:
+                    if paths[controller] == paths[0]:
+                        continue
+                    else:#paths not consistent
+                        main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+                        consistent_paths_result = main.FALSE
+
+                else:
+                    main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+                    consistent_paths_result = main.FALSE
+                    main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+            utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+                    onpass="Paths count is consistent across all ONOS nodes",
+                    onfail="ONOS nodes have different counts of paths")
+
+
+            topo_result = devices_results and ports_results and links_results\
+                    and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+        topo_result = topo_result and int(count <= 2)
+        note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+        main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
                 str(elapsed) + " seconds, " + str(count) +" tries" )
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
@@ -1000,7 +1453,7 @@
         '''
         #NOTE: You should probably run a topology check after this
 
-        link_sleep = int(main.params['timers']['LinkDiscovery'])
+        link_sleep = float(main.params['timers']['LinkDiscovery'])
 
         description = "Turn off a link to ensure that Link Discovery is working properly"
         main.log.report(description)
@@ -1022,7 +1475,7 @@
         '''
         #NOTE: You should probably run a topology check after this
 
-        link_sleep = int(main.params['timers']['LinkDiscovery'])
+        link_sleep = float(main.params['timers']['LinkDiscovery'])
 
         description = "Restore a link to ensure that Link Discovery is working properly"
         main.log.report(description)
@@ -1045,7 +1498,7 @@
         #NOTE: You should probably run a topology check after this
         import time
 
-        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+        switch_sleep = float(main.params['timers']['SwitchDiscovery'])
 
         description = "Killing a switch to ensure it is discovered correctly"
         main.log.report(description)
@@ -1054,13 +1507,18 @@
         #TODO: Make this switch parameterizable
         main.step("Kill s28 ")
         main.log.report("Deleting s28")
-        #FIXME: use new dynamic topo functions
         main.Mininet1.del_switch("s28")
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
         time.sleep(switch_sleep)
+        device = main.ONOScli1.get_device(dpid="0028")
         #Peek at the deleted switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == False:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Kill switch succesful",
+                onfail="Failed to kill switch?")
 
     def CASE12 (self, main) :
         '''
@@ -1068,7 +1526,8 @@
         '''
         #NOTE: You should probably run a topology check after this
         import time
-        #FIXME: use new dynamic topo functions
+
+        switch_sleep = float(main.params['timers']['SwitchDiscovery'])
         description = "Adding a switch to ensure it is discovered correctly"
         main.log.report(description)
         main.case(description)
@@ -1080,7 +1539,7 @@
         main.Mininet1.add_link('s28', 's3')
         main.Mininet1.add_link('s28', 's6')
         main.Mininet1.add_link('s28', 'h28')
-        main.Mininet1.assign_sw_controller(sw="28",count=7,
+        main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
                 ip1=ONOS1_ip,port1=ONOS1_port,
                 ip2=ONOS2_ip,port2=ONOS2_port,
                 ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1090,9 +1549,15 @@
                 ip7=ONOS7_ip,port7=ONOS7_port)
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
         time.sleep(switch_sleep)
-        #Peek at the added switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        device = main.ONOScli1.get_device(dpid="0028")
+        #Peek at the deleted switch
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == True:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="add switch succesful",
+                onfail="Failed to add switch?")
 
     def CASE13 (self, main) :
         '''
@@ -1100,14 +1565,41 @@
         '''
         import os
         import time
+        #printing colors to terminal
+        colors = {}
+        colors['cyan']   = '\033[96m'
+        colors['purple'] = '\033[95m'
+        colors['blue']   = '\033[94m'
+        colors['green']  = '\033[92m'
+        colors['yellow'] = '\033[93m'
+        colors['red']    = '\033[91m'
+        colors['end']    = '\033[0m'
         description = "Test Cleanup"
         main.log.report(description)
         main.case(description)
         main.step("Killing tcpdumps")
         main.Mininet2.stop_tcpdump()
 
+        main.step("Checking ONOS Logs for errors")
+        print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS1_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS2_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS3_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS4_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS5_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS6_ip)
+        print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS7_ip)
+
         main.step("Copying MN pcap and ONOS log files to test station")
         testname = main.TEST
+        teststation_user = main.params['TESTONUSER']
+        teststation_IP = main.params['TESTONIP']
         #NOTE: MN Pcap file is being saved to ~/packet_captures
         #       scp this file as MN and TestON aren't necessarily the same vm
         #FIXME: scp
@@ -1119,19 +1611,26 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
         #std*.log's
@@ -1141,24 +1640,29 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS2-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS3-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS4-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS5-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS6-"+f )
-            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS7-"+f )
 
 
-
-
         #sleep so scp can finish
         time.sleep(10)
         main.step("Packing and rotating pcap archives")
@@ -1169,3 +1673,139 @@
         utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
                 onpass="Test cleanup successful",
                 onfail="Test cleanup NOT successful")
+
+    def CASE14 ( self, main ) :
+        '''
+        start election app on all onos nodes
+        '''
+        leader_result = main.TRUE
+        #install app on onos 1
+        main.log.info("Install leadership election app")
+        main.ONOScli1.feature_install("onos-app-election")
+        #wait for election
+        #check for leader
+        leader = main.ONOScli1.election_test_leader()
+        #verify leader is ONOS1
+        if leader == ONOS1_ip:
+            #all is well
+            pass
+        elif leader == None:
+            #No leader elected
+            main.log.report("No leader was elected")
+            leader_result = main.FALSE
+        elif leader == main.FALSE:
+            #error in  response
+            #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+            main.log.report("Something is wrong with election_test_leader function, check the error logs")
+            leader_result = main.FALSE
+        else:
+            #error in  response
+            main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+            leader_result = main.FALSE
+
+
+
+
+        #install on other nodes and check for leader.
+        #Should be onos1 and each app should show the same leader
+        for controller in range(2,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            node.feature_install("onos-app-election")
+            leaderN = node.election_test_leader()
+            #verify leader is ONOS1
+            if leaderN == ONOS1_ip:
+                #all is well
+                pass
+            elif leaderN == main.FALSE:
+                #error in  response
+                #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+            elif leader != leaderN:
+                leader_result = main.FALSE
+                main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+                        " as the leader of the election app. Leader should be "+str(leader) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+    def CASE15 ( self, main ) :
+        '''
+        Check that Leadership Election is still functional
+        '''
+        leader_result = main.TRUE
+        description = "Check that Leadership Election is still functional"
+        main.log.report(description)
+        main.case(description)
+        main.step("Find current leader and withdraw")
+        leader = main.ONOScli1.election_test_leader()
+        #TODO: do some sanity checking on leader before using it
+        withdraw_result = main.FALSE
+        if leader == ONOS1_ip:
+            old_leader = getattr( main, "ONOScli1" )
+        elif leader == ONOS2_ip:
+            old_leader = getattr( main, "ONOScli2" )
+        elif leader == ONOS3_ip:
+            old_leader = getattr( main, "ONOScli3" )
+        elif leader == ONOS4_ip:
+            old_leader = getattr( main, "ONOScli4" )
+        elif leader == ONOS5_ip:
+            old_leader = getattr( main, "ONOScli5" )
+        elif leader == ONOS6_ip:
+            old_leader = getattr( main, "ONOScli6" )
+        elif leader == ONOS7_ip:
+            old_leader = getattr( main, "ONOScli7" )
+        elif leader == None or leader == main.FALSE:
+            main.log.report("Leader for the election app should be an ONOS node,"\
+                    +"instead got '"+str(leader)+"'")
+            leader_result = main.FALSE
+        withdraw_result = old_leader.election_test_withdraw()
+
+
+        main.step("Make sure new leader is elected")
+        leader_list = []
+        for controller in range(1,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            leader_list.append( node.election_test_leader() )
+        for leaderN in leader_list:
+            if leaderN == leader:
+                main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+                        " as leader after they withdrew")
+                leader_result = main.FALSE
+            elif leaderN == main.FALSE:
+                #error in  response
+                #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+        consistent_leader = main.FALSE
+        if len( set( leader_list ) ) == 1:
+            main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+            consistent_leader = main.TRUE
+        else:
+            main.log.report("Inconsistent responses for leader of Election-app:")
+            for n in range(len(leader_list)):
+                main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+
+        main.step("Run for election on old leader(just so everyone is in the hat)")
+        run_result = old_leader.election_test_run()
+        if consistent_leader == main.TRUE:
+            after_run = main.ONOScli1.election_test_leader()
+            #verify leader didn't just change
+            if after_run == leader_list[0]:
+                leader_result = main.TRUE
+            else:
+                leader_result = main.FALSE
+        #TODO: assert on  run and withdraw results?
+
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
index e1e75f9..95a75d7 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
@@ -1,9 +1,10 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
     <cellName>HA</cellName>
     </ENV>
-    <Git>True</Git>
+    <Git>False</Git>
+    <num_controllers> 1 </num_controllers>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
@@ -28,7 +29,7 @@
         <port7>6633</port7>
     </CTRL>
     <TESTONUSER>admin</TESTONUSER>
-    <TESTONIP>10.128.30.10</TESTONIP>
+    <TESTONIP>10.128.30.9</TESTONIP>
     <PING>
         <source1>h8</source1>
         <source2>h9</source2>
@@ -52,8 +53,8 @@
         <target10>10.0.0.27</target10>
     </PING>
     <timers>
-        <LinkDiscovery>2</LinkDiscovery>
-        <SwitchDiscovery>2</SwitchDiscovery>
+        <LinkDiscovery>.2</LinkDiscovery>
+        <SwitchDiscovery>.2</SwitchDiscovery>
     </timers>
     <MNtcpdump>
         <intf>eth0</intf>
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
index e6f2fde..6a6fe23 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -16,6 +16,8 @@
 CASE11: Switch down
 CASE12: Switch up
 CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
 '''
 class SingleInstanceHATestRestart:
 
@@ -61,6 +63,7 @@
         global ONOS6_port
         global ONOS7_ip
         global ONOS7_port
+        global num_controllers
 
         ONOS1_ip = main.params['CTRL']['ip1']
         ONOS1_port = main.params['CTRL']['port1']
@@ -76,6 +79,7 @@
         ONOS6_port = main.params['CTRL']['port6']
         ONOS7_ip = main.params['CTRL']['ip7']
         ONOS7_port = main.params['CTRL']['port7']
+        num_controllers = int(main.params['num_controllers'])
 
 
         main.step("Applying cell variable to environment")
@@ -112,6 +116,8 @@
                         "clean install")
         main.ONOSbench.get_version(report=True)
 
+        cell_result = main.ONOSbench.set_cell("SingleHA")
+        verify_result = main.ONOSbench.verify_cell()
         main.step("Creating ONOS package")
         package_result = main.ONOSbench.onos_package()
 
@@ -121,17 +127,18 @@
 
 
         main.step("Checking if ONOS is up yet")
-        #TODO: Refactor
-        # check bundle:list?
-        #this should be enough for ONOS to start
-        time.sleep(60)
-        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+        #TODO check bundle:list?
+        for i in range(2):
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            if onos1_isup:
+                break
         if not onos1_isup:
             main.log.report("ONOS1 didn't start!")
+
         # TODO: if it becomes an issue, we can retry this step  a few times
 
 
-        cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+        cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
 
         main.step("Start Packet Capture MN")
         main.Mininet2.start_tcpdump(
@@ -142,7 +149,7 @@
 
         case1_result = (clean_install_result and package_result and
                 cell_result and verify_result and onos1_install_result and
-                onos1_isup and cli1_results)
+                onos1_isup and cli_result)
 
         utilities.assert_equals(expect=main.TRUE, actual=case1_result,
                 onpass="Test startup successful",
@@ -186,8 +193,6 @@
                 onpass="Switch mastership assigned correctly",
                 onfail="Switches not assigned correctly to controllers")
 
-        #TODO: If assign roles is working reliably then manually 
-        #   assign mastership to the controller we want
 
 
     def CASE3(self,main) :
@@ -195,6 +200,7 @@
         Assign intents
 
         """
+        #FIXME: we must reinstall intents until we have a persistant datastore!
         import time
         import json
         import re
@@ -204,6 +210,10 @@
         main.step("Discovering  Hosts( Via pingall for now)")
         #FIXME: Once we have a host discovery mechanism, use that instead
 
+        #install onos-app-fwd
+        main.log.info("Install reactive forwarding app")
+        main.ONOScli1.feature_install("onos-app-fwd")
+
         #REACTIVE FWD test
         ping_result = main.FALSE
         time1 = time.time()
@@ -214,22 +224,30 @@
         #uninstall onos-app-fwd
         main.log.info("Uninstall reactive forwarding app")
         main.ONOScli1.feature_uninstall("onos-app-fwd")
+        #timeout for fwd flows
+        time.sleep(10)
 
         main.step("Add  host intents")
         #TODO:  move the host numbers to params
         import json
         intents_json= json.loads(main.ONOScli1.hosts())
-        intent_add_result = main.FALSE
+        intent_add_result = True
         for i in range(8,18):
             main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
             host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
             host2 =  "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
-            #NOTE: get host can return None
-            #TODO: handle this
             host1_id = main.ONOScli1.get_host(host1)['id']
             host2_id = main.ONOScli1.get_host(host2)['id']
-            tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
-            intent_add_result = intent_add_result and tmp_result
+            #NOTE: get host can return None
+            if host1_id and host2_id:
+                tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+            else:
+                main.log.error("Error, get_host() failed")
+                tmp_result = main.FALSE
+            intent_add_result = bool(intent_add_result and tmp_result)
+        utilities.assert_equals(expect=True, actual=intent_add_result,
+                onpass="Switch mastership correctly assigned",
+                onfail="Error in (re)assigning switch mastership")
         #TODO Check if intents all exist in datastore
         #NOTE: Do we need to print this once the test is working?
         #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -275,6 +293,15 @@
 
         main.step("Get the Mastership of each switch from each controller")
         global mastership_state
+        mastership_state = []
+
+        #Assert that each device has a master
+        roles_not_null = main.ONOScli1.roles_not_null()
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
         ONOS1_mastership = main.ONOScli1.roles()
         #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
         #TODO: Make this a meaningful check
@@ -285,14 +312,11 @@
         else:
             mastership_state = ONOS1_mastership
             consistent_mastership = main.TRUE
-            main.log.report("Switch roles are consistent across all ONOS nodes")
-        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
-                onpass="Switch roles are consistent across all ONOS nodes",
-                onfail="ONOS nodes have different views of switch roles")
 
 
         main.step("Get the intents from each controller")
         global intent_state
+        intent_state = []
         ONOS1_intents = main.ONOScli1.intents( json_format=True )
         intent_check = main.FALSE
         if "Error" in ONOS1_intents or not ONOS1_intents:
@@ -300,14 +324,11 @@
             main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
         else:
             intent_check = main.TRUE
-            main.log.report("Intents are consistent across all ONOS nodes")
-        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
-                onpass="Intents are consistent across all ONOS nodes",
-                onfail="ONOS nodes have different views of intents")
 
 
         main.step("Get the flows from each controller")
         global flow_state
+        flow_state = []
         ONOS1_flows = main.ONOScli1.flows( json_format=True )
         flow_check = main.FALSE
         if "Error" in ONOS1_flows or not ONOS1_flows:
@@ -317,42 +338,16 @@
             #TODO: Do a better check, maybe compare flows on switches?
             flow_state = ONOS1_flows
             flow_check = main.TRUE
-            main.log.report("Flow count is consistent across all ONOS nodes")
-        utilities.assert_equals(expect = main.TRUE,actual=flow_check,
-                onpass="The flow count is consistent across all ONOS nodes",
-                onfail="ONOS nodes have different flow counts")
 
 
         main.step("Get the OF Table entries")
         global flows
         flows=[]
         for i in range(1,29):
-            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+            flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
 
         #TODO: Compare switch flow tables with ONOS flow tables
 
-        main.step("Start continuous pings")
-        main.Mininet2.pingLong(src=main.params['PING']['source1'],
-                            target=main.params['PING']['target1'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source2'],
-                            target=main.params['PING']['target2'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source3'],
-                            target=main.params['PING']['target3'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source4'],
-                            target=main.params['PING']['target4'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source5'],
-                            target=main.params['PING']['target5'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source6'],
-                            target=main.params['PING']['target6'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source7'],
-                            target=main.params['PING']['target7'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source8'],
-                            target=main.params['PING']['target8'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source9'],
-                            target=main.params['PING']['target9'],pingTime=500)
-        main.Mininet2.pingLong(src=main.params['PING']['source10'],
-                            target=main.params['PING']['target10'],pingTime=500)
-
         main.step("Create TestONTopology object")
         ctrls = []
         count = 1
@@ -381,7 +376,7 @@
         devices_results = main.TRUE
         ports_results = main.TRUE
         links_results = main.TRUE
-        for controller in range(1): #TODO parameterize the number of controllers
+        for controller in range(num_controllers):
             if devices[controller] or not "Error" in devices[controller]:
                 current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
             else:
@@ -417,7 +412,7 @@
 
         final_assert = main.TRUE
         final_assert = final_assert and topo_result and flow_check \
-                and intent_check and consistent_mastership
+                and intent_check and consistent_mastership and roles_not_null
         utilities.assert_equals(expect=main.TRUE, actual=final_assert,
                 onpass="State check successful",
                 onfail="State check NOT successful")
@@ -436,7 +431,7 @@
 
         main.step("Checking if ONOS is up yet")
         count = 0
-        while count < 10
+        while count < 10:
             onos1_isup = main.ONOSbench.isup(ONOS1_ip)
             if onos1_isup == main.TRUE:
                 elapsed = time.time() - start
@@ -446,11 +441,12 @@
 
         cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
 
-        case_results = main.TRUE and onosi1_isup and cli_result
+        case_results = main.TRUE and onos1_isup and cli_result
         utilities.assert_equals(expect=main.TRUE, actual=case_results,
                 onpass="ONOS restart successful",
                 onfail="ONOS restart NOT successful")
-        main.log.info("ONOS took %s seconds to restart" % str(elapsed) )
+        main.log.info("ESTIMATE: ONOS took %s seconds to restart" % str(elapsed) )
+        time.sleep(5)
 
     def CASE7(self,main) :
         '''
@@ -460,6 +456,14 @@
         import json
         main.case("Running ONOS Constant State Tests")
 
+        #Assert that each device has a master
+        roles_not_null = main.ONOScli1.roles_not_null()
+        utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+                onpass="Each device has a master",
+                onfail="Some devices don't have a master assigned")
+
+
+
         main.step("Check if switch roles are consistent across all nodes")
         ONOS1_mastership = main.ONOScli1.roles()
         #FIXME: Refactor this whole case for single instance
@@ -508,13 +512,14 @@
             main.log.report("Error in getting ONOS intents")
             main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
         else:
-            intent_state = ONOS1_intents
             intent_check = main.TRUE
             main.log.report("Intents are consistent across all ONOS nodes")
         utilities.assert_equals(expect = main.TRUE,actual=intent_check,
                 onpass="Intents are consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of intents")
 
+        #NOTE: Hazelcast has no durability, so intents are lost
+        '''
         main.step("Compare current intents with intents before the failure")
         if intent_state == ONOS1_intents:
             same_intents = main.TRUE
@@ -526,6 +531,7 @@
                 onpass="Intents are consistent with before failure",
                 onfail="The Intents changed during failure")
         intent_check = intent_check and same_intents
+        '''
 
 
 
@@ -534,42 +540,48 @@
         flows2=[]
         for i in range(28):
             main.log.info("Checking flow table on s" + str(i+1))
-            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
             flows2.append(tmp_flows)
-            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            Flow_Tables = Flow_Tables and temp_result
             if Flow_Tables == main.FALSE:
                 main.log.info("Differences in flow table for switch: "+str(i+1))
-                break
         if Flow_Tables == main.TRUE:
             main.log.report("No changes were found in the flow tables")
         utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
                 onpass="No changes were found in the flow tables",
                 onfail="Changes were found in the flow tables")
 
-        main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
-        #FIXME: This check is always failing. Investigate cause
-        #NOTE:  this may be something to do with file permsissions
-        #       or slight change in format
-        main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
-        Loss_In_Pings = main.FALSE 
-        #NOTE: checkForLoss returns main.FALSE with 0% packet loss
-        for i in range(8,18):
-            main.log.info("Checking for a loss in pings along flow from s" + str(i))
-            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
-        if Loss_In_Pings == main.TRUE:
-            main.log.info("Loss in ping detected")
-        elif Loss_In_Pings == main.ERROR:
-            main.log.info("There are multiple mininet process running")
-        elif Loss_In_Pings == main.FALSE:
-            main.log.info("No Loss in the pings")
-            main.log.report("No loss of dataplane connectivity")
-        utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
-                onpass="No Loss of connectivity",
-                onfail="Loss of dataplane connectivity detected")
+
+        #Test of LeadershipElection
+
+        leader = ONOS1_ip
+        leader_result = main.TRUE
+        for controller in range(1,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            leaderN = node.election_test_leader()
+            #verify leader is ONOS1
+            #NOTE even though we restarted ONOS, it is the only one so onos 1 must be leader
+            if leaderN == leader:
+                #all is well
+                pass
+            elif leaderN == main.FALSE:
+                #error in  response
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+            elif leader != leaderN:
+                leader_result = main.FALSE
+                main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+                        " as the leader of the election app. Leader should be "+str(leader) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
 
 
-        #TODO:add topology to this or leave as a seperate case?
-        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = mastership_check and intent_check and Flow_Tables and roles_not_null\
+                and leader_result
         result = int(result)
         if result == main.TRUE:
             main.log.report("Constant State Tests Passed")
@@ -606,62 +618,62 @@
         ports_results = main.TRUE
         links_results = main.TRUE
         topo_result = main.FALSE
-        start_time = time.time()
         elapsed = 0
         count = 0
-        while topo_result == main.FALSE and elapsed < 120:
+        main.step("Collecting topology information from ONOS")
+        start_time = time.time()
+        while topo_result == main.FALSE and elapsed < 60:
             count = count + 1
-            try:
-                main.step("Collecting topology information from ONOS")
-                devices = []
-                devices.append( main.ONOScli1.devices() )
-                '''
-                hosts = []
-                hosts.append( main.ONOScli1.hosts() )
-                '''
-                ports = []
-                ports.append( main.ONOScli1.ports() )
-                links = []
-                links.append( main.ONOScli1.links() )
-                for controller in range(1): #TODO parameterize the number of controllers
-                    if devices[controller] or not "Error" in devices[controller]:
-                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
-                    else:
-                        current_devices_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
-                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+            if count > 1:
+                MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+            cli_start = time.time()
+            devices = []
+            devices.append( main.ONOScli1.devices() )
+            '''
+            hosts = []
+            hosts.append( main.ONOScli1.hosts() )
+            '''
+            ports = []
+            ports.append( main.ONOScli1.ports() )
+            links = []
+            links.append( main.ONOScli1.links() )
+            elapsed = time.time() - start_time
+            print "CLI time: " + str(time.time() - cli_start)
 
-                    if ports[controller] or not "Error" in ports[controller]:
-                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
-                    else:
-                        current_ports_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
-                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+            for controller in range(num_controllers):
+                if devices[controller] or not "Error" in devices[controller]:
+                    current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                else:
+                    current_devices_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                        onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
 
-                    if links[controller] or not "Error" in links[controller]:
-                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
-                    else:
-                        current_links_result = main.FALSE
-                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
-                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
-                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
-            except:
-                main.log.error("something went wrong in topo comparison")
-                main.log.warn( repr( devices ) )
-                main.log.warn( repr( ports ) )
-                main.log.warn( repr( links ) )
+                if ports[controller] or not "Error" in ports[controller]:
+                    current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                else:
+                    current_ports_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                        onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
 
+                if links[controller] or not "Error" in links[controller]:
+                    current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                else:
+                    current_links_result = main.FALSE
+                utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                        onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                        onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
             devices_results = devices_results and current_devices_result
             ports_results = ports_results and current_ports_result
             links_results = links_results and current_links_result
-            elapsed = time.time() - start_time
-        time_threshold = elapsed < 1
-        topo_result = devices_results and ports_results and links_results and time_threshold
-        #TODO make sure this step is non-blocking. IE add a timeout
-        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+            topo_result = devices_results and ports_results and links_results
+
+        topo_result = topo_result and int(count <= 2)
+        main.log.report("Very crass estimate for topology discovery/convergence(note it takes about 1 seconds to read the topology from each ONOS instance): " +\
                 str(elapsed) + " seconds, " + str(count) +" tries" )
+        if elapsed > 60:
+            main.log.report("Giving up on topology convergence")
         utilities.assert_equals(expect=main.TRUE, actual=topo_result,
                 onpass="Topology Check Test successful",
                 onfail="Topology Check Test NOT successful")
@@ -675,7 +687,7 @@
         '''
         #NOTE: You should probably run a topology check after this
 
-        link_sleep = int(main.params['timers']['LinkDiscovery'])
+        link_sleep = float(main.params['timers']['LinkDiscovery'])
 
         description = "Turn off a link to ensure that Link Discovery is working properly"
         main.log.report(description)
@@ -697,7 +709,7 @@
         '''
         #NOTE: You should probably run a topology check after this
 
-        link_sleep = int(main.params['timers']['LinkDiscovery'])
+        link_sleep = float(main.params['timers']['LinkDiscovery'])
 
         description = "Restore a link to ensure that Link Discovery is working properly"
         main.log.report(description)
@@ -720,7 +732,7 @@
         #NOTE: You should probably run a topology check after this
         import time
 
-        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+        switch_sleep = float(main.params['timers']['SwitchDiscovery'])
 
         description = "Killing a switch to ensure it is discovered correctly"
         main.log.report(description)
@@ -729,13 +741,18 @@
         #TODO: Make this switch parameterizable
         main.step("Kill s28 ")
         main.log.report("Deleting s28")
-        #FIXME: use new dynamic topo functions
         main.Mininet1.del_switch("s28")
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
         time.sleep(switch_sleep)
+        device = main.ONOScli1.get_device(dpid="0028")
         #Peek at the deleted switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == False:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Kill switch succesful",
+                onfail="Failed to kill switch?")
 
     def CASE12 (self, main) :
         '''
@@ -743,7 +760,8 @@
         '''
         #NOTE: You should probably run a topology check after this
         import time
-        #FIXME: use new dynamic topo functions
+
+        switch_sleep = float(main.params['timers']['SwitchDiscovery'])
         description = "Adding a switch to ensure it is discovered correctly"
         main.log.report(description)
         main.case(description)
@@ -759,9 +777,15 @@
                 ip1=ONOS1_ip,port1=ONOS1_port)
         main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
         time.sleep(switch_sleep)
-        #Peek at the added switch
-        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
-        #TODO do some sort of check here
+        device = main.ONOScli1.get_device(dpid="0028")
+        #Peek at the deleted switch
+        main.log.warn( str(device) )
+        result = main.FALSE
+        if device and device['available'] == True:
+            result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="add switch succesful",
+                onfail="Failed to add switch?")
 
     def CASE13 (self, main) :
         '''
@@ -769,14 +793,28 @@
         '''
         import os
         import time
+        #printing colors to terminal
+        colors = {}
+        colors['cyan']   = '\033[96m'
+        colors['purple'] = '\033[95m'
+        colors['blue']   = '\033[94m'
+        colors['green']  = '\033[92m'
+        colors['yellow'] = '\033[93m'
+        colors['red']    = '\033[91m'
+        colors['end']    = '\033[0m'
         description = "Test Cleanup"
         main.log.report(description)
         main.case(description)
         main.step("Killing tcpdumps")
         main.Mininet2.stop_tcpdump()
 
+        main.step("Checking ONOS Logs for errors")
+        print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+        print main.ONOSbench.check_logs(ONOS1_ip)
         main.step("Copying MN pcap and ONOS log files to test station")
         testname = main.TEST
+        teststation_user = main.params['TESTONUSER']
+        teststation_IP = main.params['TESTONIP']
         #NOTE: MN Pcap file is being saved to ~/packet_captures
         #       scp this file as MN and TestON aren't necessarily the same vm
         #FIXME: scp
@@ -788,8 +826,11 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
+            main.ONOSbench.handle.expect("\$")
+            print main.ONOSbench.handle.before
 
         #std*.log's
         #NOTE: must end in /
@@ -798,7 +839,8 @@
         #NOTE: must end in /
         dst_dir = "~/packet_captures/"
         for f in log_files:
-            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+            main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+                    teststation_user +"@"+teststation_IP+":"+\
                     dst_dir + str(testname) + "-ONOS1-"+f )
 
 
@@ -812,3 +854,117 @@
         utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
                 onpass="Test cleanup successful",
                 onfail="Test cleanup NOT successful")
+
+    def CASE14 ( self, main ) :
+        '''
+        start election app on all onos nodes
+        '''
+        leader_result = main.TRUE
+        #install app on onos 1
+        main.log.info("Install leadership election app")
+        main.ONOScli1.feature_install("onos-app-election")
+        #wait for election
+        #check for leader
+        leader = main.ONOScli1.election_test_leader()
+        #verify leader is ONOS1
+        if leader == ONOS1_ip:
+            #all is well
+            pass
+        elif leader == None:
+            #No leader elected
+            main.log.report("No leader was elected")
+            leader_result = main.FALSE
+        elif leader == main.FALSE:
+            #error in  response
+            #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+            main.log.report("Something is wrong with election_test_leader function, check the error logs")
+            leader_result = main.FALSE
+        else:
+            #error in  response
+            main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+            leader_result = main.FALSE
+
+
+
+
+        #install on other nodes and check for leader.
+        #Should be onos1 and each app should show the same leader
+        for controller in range(2,num_controllers+1):
+            node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+            node.feature_install("onos-app-election")
+            leaderN = node.election_test_leader()
+            #verify leader is ONOS1
+            if leaderN == ONOS1_ip:
+                #all is well
+                pass
+            elif leaderN == main.FALSE:
+                #error in  response
+                #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+                main.log.report("Something is wrong with election_test_leader function, check the error logs")
+                leader_result = main.FALSE
+            elif leader != leaderN:
+                leader_result = main.FALSE
+                main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+                        " as the leader of the election app. Leader should be "+str(leader) )
+        if leader_result:
+            main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+    def CASE15 ( self, main ) :
+        '''
+        Check that Leadership Election is still functional
+        '''
+        leader_result = main.TRUE
+        description = "Check that Leadership Election is still functional"
+        main.log.report(description)
+        main.case(description)
+        main.step("Find current leader and withdraw")
+        leader = main.ONOScli1.election_test_leader()
+        #TODO: do some sanity checking on leader before using it
+        withdraw_result = main.FALSE
+        if leader == ONOS1_ip:
+            old_leader = getattr( main, "ONOScli1" )
+        elif leader == None or leader == main.FALSE:
+            main.log.report("Leader for the election app should be an ONOS node,"\
+                    +"instead got '"+str(leader)+"'")
+            leader_result = main.FALSE
+        withdraw_result = old_leader.election_test_withdraw()
+
+
+        main.step("Make sure new leader is elected")
+        leader_list = []
+        leaderN =  main.ONOScli1.election_test_leader() 
+        if leaderN == leader:
+            main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+                    " as leader after they withdrew")
+            leader_result = main.FALSE
+        elif leaderN == main.FALSE:
+            #error in  response
+            #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+            main.log.report("Something is wrong with election_test_leader function, check the error logs")
+            leader_result = main.FALSE
+        elif leaderN == None:
+            main.log.info("There is no leader after the app withdrew from election")
+        if leader_result:
+            main.log.report("Leadership election tests passed(There is no leader after the old leader resigned)")
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="Something went wrong with Leadership election")
+
+
+        main.step("Run for election on old leader(just so everyone is in the hat)")
+        run_result = old_leader.election_test_run()
+        leader = main.ONOScli1.election_test_leader()
+        #verify leader is ONOS1
+        if leader == ONOS1_ip:
+            leader_result = main.TRUE
+        else:
+            leader_result = main.FALSE
+        #TODO: assert on  run and withdraw results?
+
+        utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+                onpass="Leadership election passed",
+                onfail="ONOS1's election app was not leader after it re-ran for election")
+