diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 44a70b8..9c5ad4d 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -42,7 +42,7 @@
             
         logmsg = logmsg + "\n\tTest Script :" + path + "Tests/" + main.TEST + ".py"+ ""
         logmsg = logmsg + "\n\tTest Params : " + path + "Tests/" + main.TEST + ".params" + ""
-        logmsg = logmsg + "\n\tTopology : " + path + "Tests/" +main.TEST + ".tpl" + ""
+        logmsg = logmsg + "\n\tTopology : " + path + "Tests/" +main.TEST + ".topo" + ""
         logmsg = logmsg + "\n"+" " * 30+"+" +"-" * 18+"+" +"\n" +"-" * 27+"  { Script Exec Params }  "+"-" * 27 +"\n" +" " * 30 +"+"+"-" * 18 +"+\n";
         values = "\n\t" + str(main.params)
         values = re.sub(",", "\n\t", values)
@@ -110,6 +110,7 @@
            
         main.LogFileName = main.logdir + "/" + main.TEST + "_" +str(currentTime) + ".log"
         main.ReportFileName = main.logdir + "/" + main.TEST + "_" + str(currentTime) + ".rpt"
+        main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
                 
         #### Add log-level - Report
         logging.addLevelName(9, "REPORT")
@@ -228,7 +229,17 @@
         #main.log.report(testResult)
         main.testResult = testResult
         main.log.exact(testResult)
-                
+
+        ##CSV output needed for Jenkin's plot plugin
+        #NOTE: the elements were orded based on the colors assigned to the data
+        logfile = open(main.JenkinsCSV ,"w")
+        logfile.write(",".join( ['Tests Failed', 'Tests Passed', 'Tests Planned'] ) + "\n")
+        logfile.write(",".join( [str(main.TOTAL_TC_FAIL), str(main.TOTAL_TC_PASS), str(main.TOTAL_TC_PLANNED)] ))
+        logfile.close()
+
+
+
+
     def updateCaseResults(self,main):
         '''
             Update the case result based on the steps execution and asserting each step in the test-case
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
new file mode 100755
index 0000000..235d138
--- /dev/null
+++ b/TestON/dependencies/rotate.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+
+# NOTE: Taken fnd modified from onos.sh
+# pack-rotate-log [packname] "[log-filenames]" [max rotations]
+# Note: [packname] and all the log-files specified by [log-filenames]
+#       must reside in same dir
+# Example:
+#  pack="/foo/bar/testlogs"
+#  logfiles="/foo/bar/test1.log /foo/bar/test*.log"
+#  pack-rotate-log $pack "$logfiles" 5
+#   => testlogs.tar.bz2 (will contain test1.log test2.log ...)
+#      testlogs.tar.bz2 -> testlogs.1.tar.bz2
+#      testlogs.1.tar.bz2 -> testlogs.2.tar.bz2
+#      ...
+function pack-rotate-log {
+  local packname=$1
+  local logfiles=$2
+  local nr_max=${3:-10}
+  local suffix=".tar.bz2"
+
+  # rotate
+  for i in `seq $(expr $nr_max - 1) -1 1`; do
+    if [ -f ${packname}.${i}${suffix} ]; then
+      mv -f -- ${packname}.${i}${suffix} ${packname}.`expr $i + 1`${suffix}
+    fi
+  done
+  if [ -f ${packname}${suffix} ]; then
+    mv -- ${packname}${suffix} ${packname}.1${suffix}
+  fi
+
+  # pack
+  local existing_logfiles=$( ls -1 $logfiles  2>/dev/null | xargs -n1  basename 2>/dev/null)
+  if [ ! -z "${existing_logfiles}" ]; then
+    tar cjf ${packname}${suffix} -C `dirname ${packname}` -- ${existing_logfiles}
+    for word in ${existing_logfiles}
+    do
+        rm -- `dirname ${packname}`/${word}
+    done
+   fi
+}
+
+
+
+#Begin script
+#NOTE: This seems to break the TestON summary since it mentions the testname
+#echo "Rotating logs for '${1}' test"
+base_name=$1
+root_dir="/home/admin/packet_captures"
+timestamp=`date +%Y_%B_%d_%H_%M_%S`
+#Maybe this should be an argument? pack-and-rotate supports that
+nr_max=20
+
+pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap" ${nr_max}
diff --git a/TestON/drivers/common/cli/emulator/lincoedriver.py b/TestON/drivers/common/cli/emulator/lincoedriver.py
index 4d2cc01..6f7ce3a 100644
--- a/TestON/drivers/common/cli/emulator/lincoedriver.py
+++ b/TestON/drivers/common/cli/emulator/lincoedriver.py
@@ -293,8 +293,8 @@
         '''
         try:
             #Send CTRL+C twice to exit CLI
-            self.handle.sendline("\x03")
-            self.handle.sendline("\x03")
+            self.handle.send("\x03")
+            self.handle.send("\x03")
             self.handle.expect("\$")
 
         except pexpect.EOF:
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index c888e23..2f31b7d 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -146,14 +146,18 @@
         topoDict = self.num_switches_n_links(*topoArgList)
         return topoDict
 
-    def pingall(self):
+    def pingall(self, timeout=300):
         '''
         Verifies the reachability of the hosts using pingall command.
+        Optional parameter timeout allows you to specify how long to wait for pingall to complete
+        Returns:
+                main.TRUE if pingall completes with no pings dropped
+                otherwise main.FALSE
         '''
         if self.handle :
             main.log.info(self.name+": Checking reachabilty to the hosts using pingall")
             try:
-                response = self.execute(cmd="pingall",prompt="mininet>",timeout=300)
+                response = self.execute(cmd="pingall",prompt="mininet>",timeout=int(timeout))
             except pexpect.EOF:
                 main.log.error(self.name + ": EOF exception found")
                 main.log.error(self.name + ":     " + self.handle.before)
@@ -656,6 +660,187 @@
         else:
             main.log.info(response)
 
+    def add_switch( self, sw, **kwargs ):
+        '''
+        adds a switch to the mininet topology
+        NOTE: this uses a custom mn function
+        NOTE: cannot currently specify what type of switch
+        required params:
+            switchname = name of the new switch as a string
+        optional keyvalues:
+            dpid = "dpid"
+        returns: main.FASLE on an error, else main.TRUE
+        '''
+        dpid = kwargs.get('dpid', '')
+        command = "addswitch " + str( sw ) + " " + str( dpid )
+        try:
+            response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+            if re.search("already exists!", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("Error", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("usage:", response):
+                main.log.warn(response)
+                return main.FALSE
+            else:
+                return main.TRUE
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":     " + self.handle.before)
+            main.cleanup()
+            main.exit()
+
+    def del_switch( self, sw ):
+        '''
+        delete a switch from the mininet topology
+        NOTE: this uses a custom mn function
+        required params:
+            switchname = name of the switch as a string
+        returns: main.FASLE on an error, else main.TRUE
+        '''
+        command = "delswitch " + str( sw )
+        try:
+            response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+            if re.search("no switch named", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("Error", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("usage:", response):
+                main.log.warn(response)
+                return main.FALSE
+            else:
+                return main.TRUE
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":     " + self.handle.before)
+            main.cleanup()
+            main.exit()
+
+    def add_link( self, node1, node2 ):
+        '''
+        add a link to the mininet topology
+        NOTE: this uses a custom mn function
+        NOTE: cannot currently specify what type of link
+        required params:
+            node1 = the string node name of the first endpoint of the link
+            node2 = the string node name of the second endpoint of the link
+        returns: main.FASLE on an error, else main.TRUE
+        '''
+        command = "addlink " + str( node1 ) + " " + str( node2 )
+        try:
+            response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+            if re.search("doesnt exist!", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("Error", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("usage:", response):
+                main.log.warn(response)
+                return main.FALSE
+            else:
+                return main.TRUE
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":     " + self.handle.before)
+            main.cleanup()
+            main.exit()
+
+    def del_link( self, node1, node2 ):
+        '''
+        delete a link from the mininet topology
+        NOTE: this uses a custom mn function
+        required params:
+            node1 = the string node name of the first endpoint of the link
+            node2 = the string node name of the second endpoint of the link
+        returns: main.FASLE on an error, else main.TRUE
+        '''
+        command = "dellink " + str( node1 ) + " " + str( node2 )
+        try:
+            response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+            if re.search("no node named", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("Error", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("usage:", response):
+                main.log.warn(response)
+                return main.FALSE
+            else:
+                return main.TRUE
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":     " + self.handle.before)
+            main.cleanup()
+            main.exit()
+
+    def add_host( self, hostname, **kwargs ):
+        '''
+        Add a host to the mininet topology
+        NOTE: this uses a custom mn function
+        NOTE: cannot currently specify what type of host
+        required params:
+            hostname = the string hostname
+        optional key-value params
+            switch = "switch name"
+            returns: main.FASLE on an error, else main.TRUE
+        '''
+        switch = kwargs.get('switch', '')
+        command = "addhost " + str( hostname ) + " " + str( switch )
+        try:
+            response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+            if re.search("already exists!", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("doesnt exists!", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("Error", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("usage:", response):
+                main.log.warn(response)
+                return main.FALSE
+            else:
+                return main.TRUE
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":     " + self.handle.before)
+            main.cleanup()
+            main.exit()
+
+    def del_host( self, hostname ):
+        '''
+        delete a host from the mininet topology
+        NOTE: this uses a custom mn function
+        required params:
+            hostname = the string hostname
+            returns: main.FASLE on an error, else main.TRUE
+        '''
+        command = "delhost " + str( hostname )
+        try:
+            response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+            if re.search("no host named", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("Error", response):
+                main.log.warn(response)
+                return main.FALSE
+            elif re.search("usage:", response):
+                main.log.warn(response)
+                return main.FALSE
+            else:
+                return main.TRUE
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":     " + self.handle.before)
+            main.cleanup()
+            main.exit()
 
     def disconnect(self):
         main.log.info(self.name+": Disconnecting mininet...")
@@ -928,10 +1113,8 @@
             onos_ports = [x for x in onos_ports]
 
             #TODO: handle other reserved port numbers besides LOCAL
-            #NOTE: List of Reserved Ports
-            #   Local port: -2 in unsigned int in Openflow, the size depends on
-            #       openflow version( 1.0 is 16 bit, 1.3 is 32 bit), ONOS shows
-            #       'local', we store as long(uint64(-2))
+            #NOTE: Reserved ports
+            #   Local port: -2 in Openflow, ONOS shows 'local', we store as long(uint64(-2))
             for mn_port in mn_ports_log:
                 if mn_port in onos_ports:
                     #don't set results to true here as this is just one of many checks and it might override a failure
@@ -951,7 +1134,7 @@
                 switch_result = main.FALSE
                 main.log.warn("Ports in ONOS but not MN: " + str(onos_ports) )
             if switch_result == main.FALSE:
-                main.log.report("The list of active ports for switch %s(%s) does not match:" % (mn_switch['name'], mn_switch['dpid']) )
+                main.log.report("The list of ports for switch %s(%s) does not match:" % (mn_switch['name'], mn_switch['dpid']) )
                 main.log.warn("mn_ports[]  =  " + str(mn_ports_log))
                 main.log.warn("onos_ports[] = " + str(onos_ports_log))
             ports_results = ports_results and switch_result
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 5d2a7a3..1795a16 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -80,9 +80,16 @@
         Returns main.ERROR if "found multiple mininet" is found and
         Returns main.TRUE else
         '''
+        #TODO: maybe we want to return the % loss instead? This way we can set an acceptible loss %.
+        #EX: 393 packets transmitted, 380 received, 3% packet loss, time 78519ms
+        # we may need to return a float to get around rounding errors
+
         import os
         self.handle.sendline("")
         self.handle.expect("\$")
+        #Clear any output waiting in the bg from killing pings
+        self.handle.sendline("")
+        self.handle.expect("\$")
         self.handle.sendline("cat " + pingList)
         self.handle.expect(pingList)
         self.handle.expect("\$")
@@ -162,8 +169,13 @@
         self.handle.sendline("")
         self.handle.expect("\$")
         self.handle.sendline("")
-        self.handle.expect("\$")
-        return main.TRUE
+        i=self.handle.expect(["password","\$"])
+        if i == 0:
+            main.log.error("Error, sudo asking for password")
+            main.log.error(self.handle.before)
+            return main.FALSE
+        else:
+            return main.TRUE
     
     def pingLongKill(self):
         import time
@@ -215,34 +227,6 @@
             main.last_result = main.FALSE
             return main.FALSE
 
-
-    def pingall(self):
-        '''
-        Verifies the reachability of the hosts using pingall command.
-        This function is required by Packey Optical test
-        '''
-        if self.handle :
-            main.log.info(self.name+": Checking reachabilty to the hosts using pingall")
-            try:
-                response = self.execute(cmd="pingall",prompt="mininet>",timeout=120)
-                print "response: " + str(response)
-            except pexpect.EOF:
-                main.log.error(self.name + ": EOF exception found")
-                main.log.error(self.name + ":     " + self.handle.before)
-            pattern = 'Results\:\s0\%\sdropped\s'
-            if re.search(pattern,response):
-                main.log.info(self.name+": All hosts are reachable")
-                return main.TRUE
-            else:
-                main.log.error(self.name+": Unable to reach all the hosts")
-                return main.FALSE
-        else :
-            main.log.error(self.name+": Connection failed to the host")
-            return main.FALSE
-
-
-
- 
     def pingHost(self,**pingParams):
         ''' 
         Pings between two hosts on remote mininet  
@@ -292,154 +276,14 @@
         else :
             main.log.error("Connection failed to the host") 
 
-    def ctrl_none(self):
-        '''
-        Sets all the switches to no controllers. 
-        '''
-        self.execute(cmd="~/ONOS/scripts/test-ctrl-none.sh", prompt="\$",timeout=10)
-
-    def ctrl_one(self, ip):
-        '''
-        Sets all the switches to point to the supplied IP
-        '''
-        self.execute(cmd="~/ONOS/scripts/test-ctrl-one.sh "+ip, prompt="\$",timeout=10)
- 
-    def ctrl_local(self):
-        '''
-        Sets all the switches to point to the Controller on the same machine that they are running on. 
-        '''
-        self.execute(cmd="~/ONOS/scripts/test-ctrl-local.sh ", prompt="\$",timeout=10)
-
- #   def verifySSH(self,**connectargs):
- #       response = self.execute(cmd="h1 /usr/sbin/sshd -D&",prompt="mininet>",timeout=10)
- #       response = self.execute(cmd="h4 /usr/sbin/sshd -D&",prompt="mininet>",timeout=10)
- #       for key in connectargs:
- #           vars(self)[key] = connectargs[key]
- #       response = self.execute(cmd="xterm h1 h4 ",prompt="mininet>",timeout=10)
- #       import time
- #       time.sleep(20)
- #       if self.flag == 0:
- #           self.flag = 1
- #           return main.FALSE
- #       else :
- #           return main.TRUE
- #   
- #   def getMacAddress(self,host):
- #       '''
- #           Verifies the host's ip configured or not.
- #       '''
- #       if self.handle :
- #           response = self.execute(cmd=host+" ifconfig",prompt="mininet>",timeout=10)
-
- #           pattern = "HWaddr\s(((\d|\w)+:)+(\d|\w))"
- #           mac_address_search = re.search(pattern, response)
- #           main.log.info("Mac-Address of Host "+host +" is "+mac_address_search.group(1))
- #           return mac_address_search.group(1)
- #       else :
- #           main.log.error("Connection failed to the host") 
- #   def getIPAddress(self,host):
- #       '''
- #           Verifies the host's ip configured or not.
- #       '''
- #       if self.handle :
- #           response = self.execute(cmd=host+" ifconfig",prompt="mininet>",timeout=10)
-
- #           pattern = "inet\saddr:(\d+\.\d+\.\d+\.\d+)"
- #           ip_address_search = re.search(pattern, response)
- #           main.log.info("IP-Address of Host "+host +" is "+ip_address_search.group(1))
- #           return ip_address_search.group(1)
- #       else :
- #           main.log.error("Connection failed to the host") 
- #       
- #   def dump(self):
- #       main.log.info("Dump node info")
- #       self.execute(cmd = 'dump',prompt = 'mininet>',timeout = 10)
- #       return main.TRUE
- #           
- #   def intfs(self):
- #       main.log.info("List interfaces")
- #       self.execute(cmd = 'intfs',prompt = 'mininet>',timeout = 10)
- #       return main.TRUE
- #   
- #   def net(self):
- #       main.log.info("List network connections")
- #       self.execute(cmd = 'net',prompt = 'mininet>',timeout = 10)
- #       return main.TRUE
- #   
- #   def iperf(self):
- #       main.log.info("Simple iperf TCP test between two (optionally specified) hosts")
- #       self.execute(cmd = 'iperf',prompt = 'mininet>',timeout = 10)
- #       return main.TRUE
- #   
- #   def iperfudp(self):
- #       main.log.info("Simple iperf TCP test between two (optionally specified) hosts")
- #       self.execute(cmd = 'iperfudp',prompt = 'mininet>',timeout = 10)
- #       return main.TRUE
- #   
- #   def nodes(self):
- #       main.log.info("List all nodes.")
- #       self.execute(cmd = 'nodes',prompt = 'mininet>',timeout = 10)    
- #       return main.TRUE
- #   
- #   def pingpair(self):
- #       main.log.infoe("Ping between first two hosts")
- #       self.execute(cmd = 'pingpair',prompt = 'mininet>',timeout = 20)
- #       
- #       if utilities.assert_matches(expect='0% packet loss',actual=response,onpass="No Packet loss",onfail="Hosts not reachable"):
- #           main.log.info("Ping between two hosts SUCCESS")
- #           main.last_result = main.TRUE 
- #           return main.TRUE
- #       else :
- #           main.log.error("PACKET LOST, HOSTS NOT REACHABLE")
- #           main.last_result = main.FALSE
- #           return main.FALSE
- #   
- #   def link(self,**linkargs):
- #       '''
- #       Bring link(s) between two nodes up or down
- #       '''
- #       main.log.info('Bring link(s) between two nodes up or down')
- #       args = utilities.parse_args(["END1","END2","OPTION"],**linkargs)
- #       end1 = args["END1"] if args["END1"] != None else ""
- #       end2 = args["END2"] if args["END2"] != None else ""
- #       option = args["OPTION"] if args["OPTION"] != None else ""
- #       command = "link "+str(end1) + " " + str(end2)+ " " + str(option)
- #       response = self.execute(cmd=command,prompt="mininet>",timeout=10)
- #       return main.TRUE
- #       
-
- #   def dpctl(self,**dpctlargs):
- #       '''
- #        Run dpctl command on all switches.
- #       '''
- #       main.log.info('Run dpctl command on all switches')
- #       args = utilities.parse_args(["CMD","ARGS"],**dpctlargs)
- #       cmd = args["CMD"] if args["CMD"] != None else ""
- #       cmdargs = args["ARGS"] if args["ARGS"] != None else ""
- #       command = "dpctl "+cmd + " " + str(cmdargs)
- #       response = self.execute(cmd=command,prompt="mininet>",timeout=10)
- #       return main.TRUE
- #  
- #       
- #   def get_version(self):
- #       file_input = path+'/lib/Mininet/INSTALL'
- #       version = super(Mininet, self).get_version()
- #       pattern = 'Mininet\s\w\.\w\.\w\w*'
- #       for line in open(file_input,'r').readlines():
- #           result = re.match(pattern, line)
- #           if result:
- #               version = result.group(0)
- #               
- #           
- #       return version    
-    def start_tcpdump(self, filename, intf = "eth0", port = "port 6633"):
+    def start_tcpdump(self, filename, intf = "eth0", port = "port 6633", user="admin"):
         ''' 
         Runs tpdump on an intferface and saves the file
         intf can be specified, or the default eth0 is used
         '''
         try:
             self.handle.sendline("")
-            self.handle.sendline("sudo tcpdump -n -i "+ intf + " " + port + " -w " + filename.strip() + "  &")
+            self.handle.sendline("sudo tcpdump -n -i "+ intf + " " + port + " -w " + filename.strip() + " -Z " + user + "  &")
             self.handle.sendline("")
             self.handle.sendline("")
             i=self.handle.expect(['No\ssuch\device','listening\son',pexpect.TIMEOUT,"\$"],timeout=10)
@@ -508,49 +352,6 @@
             main.log.error(self.name + ": EOF exception found")
             main.log.error(self.name + ":     " + self.handle.before)
             return main.FALSE
-    
-    def del_switch(self,sw):
-        self.handle.sendline("")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl del-br "+sw)
-        self.handle.expect("\$")
-        return main.TRUE
-
-    def add_switch(self,sw):
-        #FIXME: Remove hardcoded number of ports
-        self.handle.sendline("")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-br "+sw)
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth1")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth2")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth3")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth4")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth5")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth6")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth7")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth8")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth9")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth10")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth11")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth12")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth13")
-        self.handle.expect("\$")
-        self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth14")
-        self.handle.expect("\$")
-
 
     def disconnect(self):
         '''    
@@ -574,8 +375,6 @@
         #FIXME: clean up print statements and such
         self.handle.sendline("cd")
         self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
-        #print "cd expect status: "
-        #print self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
         #TODO: Write seperate versions of the function for this, possibly a string that tells it which switch is in use?
         #For 1.0 version of OVS
         #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $6 $7 }' |sort -n -k1"
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index feca717..4916ba5 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -90,7 +90,7 @@
                 self.handle.expect("Confirm")
                 self.handle.sendline("yes")
                 self.handle.expect("\$")
-            self.handle.sendline("\n")
+            self.handle.sendline("")
             self.handle.expect("\$")
             self.handle.sendline("exit")
             self.handle.expect("closed")
@@ -194,13 +194,7 @@
             else:
                 #If failed, send ctrl+c to process and try again
                 main.log.info("Starting CLI failed. Retrying...")
-                self.handle.sendline("\x03")
-                i = self.handle.expect(["onos>",pexpect.TIMEOUT],
-                        timeout=30)
-                #Send ctrl+d to exit the onos> prompt that was
-                #not successful
-                self.handle.sendline("\x04")
-                self.handle.expect("\$")
+                self.handle.send("\x03")
                 self.handle.sendline("onos -w "+str(ONOS_ip))
                 i = self.handle.expect(["onos>",pexpect.TIMEOUT],
                         timeout=30)
@@ -467,27 +461,21 @@
             main.log.info(self.name+" ::::::")
             main.cleanup()
             main.exit()
-        
-    def devices(self, json_format=True, grep_str=""):
+
+    def devices(self, json_format=True):
         '''
         Lists all infrastructure devices or switches
         Optional argument:
-            * grep_str - pass in a string to grep
+            * json_format - boolean indicating if you want output in json
         '''
         try:
             self.handle.sendline("")
             self.handle.expect("onos>")
-            
+
             if json_format:
-                if not grep_str:
-                    self.handle.sendline("devices -j")
-                    self.handle.expect("devices -j")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("devices -j | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("devices -j | grep '"+str(grep_str)+"'")
-                    self.handle.expect("onos>")
+                self.handle.sendline("devices -j")
+                self.handle.expect("devices -j")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 '''
                 handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
@@ -503,13 +491,8 @@
                 #print "repr(handle1) = ", repr(handle1)
                 return handle1
             else:
-                if not grep_str:
-                    self.handle.sendline("devices")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("devices | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("onos>")
+                self.handle.sendline("devices")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 #print "handle =",handle
                 return handle
@@ -525,34 +508,28 @@
             main.cleanup()
             main.exit()
 
-    def links(self, json_format=True, grep_str=""):
+    def links(self, json_format=True):
         '''
         Lists all core links
         Optional argument:
-            * grep_str - pass in a string to grep
+            * json_format - boolean indicating if you want output in json
         '''
         try:
             self.handle.sendline("")
             self.handle.expect("onos>")
-            
+
             if json_format:
-                if not grep_str:
-                    self.handle.sendline("links -j")
-                    self.handle.expect("links -j")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("links -j | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("links -j | grep '"+str(grep_str)+"'")
-                    self.handle.expect("onos>")
+                self.handle.sendline("links -j")
+                self.handle.expect("links -j")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 '''
                 handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
                 To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
                 In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
-                So we take off that escape sequence using 
+                So we take off that escape sequence using
                 ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
-                handle1 = ansi_escape.sub('', handle) 
+                handle1 = ansi_escape.sub('', handle)
                 '''
                 #print "repr(handle) =", repr(handle)
                 ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -560,13 +537,8 @@
                 #print "repr(handle1) = ", repr(handle1)
                 return handle1
             else:
-                if not grep_str:
-                    self.handle.sendline("links")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("links | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("onos>")
+                self.handle.sendline("links")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 #print "handle =",handle
                 return handle
@@ -583,34 +555,28 @@
             main.exit()
 
 
-    def ports(self, json_format=True, grep_str=""):
+    def ports(self, json_format=True):
         '''
         Lists all ports
         Optional argument:
-            * grep_str - pass in a string to grep
+            * json_format - boolean indicating if you want output in json
         '''
         try:
             self.handle.sendline("")
             self.handle.expect("onos>")
-            
+
             if json_format:
-                if not grep_str:
-                    self.handle.sendline("ports -j")
-                    self.handle.expect("ports -j")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("ports -j | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("ports -j | grep '"+str(grep_str)+"'")
-                    self.handle.expect("onos>")
+                self.handle.sendline("ports -j")
+                self.handle.expect("ports -j")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 '''
                 handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
                 To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
                 In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
-                So we take off that escape sequence using the following commads: 
+                So we take off that escape sequence using the following commads:
                 ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
-                handle1 = ansi_escape.sub('', handle) 
+                handle1 = ansi_escape.sub('', handle)
                 '''
                 #print "repr(handle) =", repr(handle)
                 ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -619,20 +585,13 @@
                 return handle1
 
             else:
-                if not grep_str:
-                    self.handle.sendline("ports")
-                    self.handle.expect("onos>")
-                    self.handle.sendline("")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("ports | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("onos>")
-                    self.handle.sendline("")
-                    self.handle.expect("onos>")
+                self.handle.sendline("ports")
+                self.handle.expect("onos>")
+                self.handle.sendline("")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 #print "handle =",handle
-                return handle  
+                return handle
         except pexpect.EOF:
             main.log.error(self.name + ": EOF exception found")
             main.log.error(self.name + ":    " + self.handle.before)
@@ -646,34 +605,32 @@
             main.exit()
 
 
-    def roles(self, json_format=True, grep_str=""):
+    def roles(self, json_format=True):
         '''
         Lists all devices and the controllers with roles assigned to them
         Optional argument:
-            * grep_str - pass in a string to grep
+            * json_format - boolean indicating if you want output in json
         '''
         try:
             self.handle.sendline("")
             self.handle.expect("onos>")
-            
+
             if json_format:
-                if not grep_str:
-                    self.handle.sendline("roles -j")
-                    self.handle.expect("roles -j")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("roles -j | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("roles -j | grep '"+str(grep_str)+"'")
-                    self.handle.expect("onos>")
+                self.handle.sendline("roles -j")
+                self.handle.expect("roles -j")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 '''
-                handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
-                To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
-                In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
-                So we take off that escape sequence using the following commads: 
+                handle variable here contains some ANSI escape color code sequences at the
+                end which are invisible in the print command output. To make that escape
+                sequence visible, use repr() function. The repr(handle) output when printed
+                shows the ANSI escape sequences. In json.loads(somestring), this somestring
+                variable is actually repr(somestring) and json.loads would fail with the escape
+                sequence.
+
+                So we take off that escape sequence using the following commads:
                 ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
-                handle1 = ansi_escape.sub('', handle) 
+                handle1 = ansi_escape.sub('', handle)
                 '''
                 #print "repr(handle) =", repr(handle)
                 ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -682,20 +639,13 @@
                 return handle1
 
             else:
-                if not grep_str:
-                    self.handle.sendline("roles")
-                    self.handle.expect("onos>")
-                    self.handle.sendline("")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("roles | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("onos>")
-                    self.handle.sendline("")
-                    self.handle.expect("onos>")
+                self.handle.sendline("roles")
+                self.handle.expect("onos>")
+                self.handle.sendline("")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 #print "handle =",handle
-                return handle  
+                return handle
         except pexpect.EOF:
             main.log.error(self.name + ": EOF exception found")
             main.log.error(self.name + ":    " + self.handle.before)
@@ -783,35 +733,29 @@
             main.log.info(self.name+" ::::::")
             main.cleanup()
             main.exit()
-    
-    def hosts(self, json_format=True, grep_str=""):
+
+    def hosts(self, json_format=True):
         '''
-        Lists all discovered hosts 
+        Lists all discovered hosts
         Optional argument:
-            * grep_str - pass in a string to grep
+            * json_format - boolean indicating if you want output in json
         '''
         try:
             self.handle.sendline("")
             self.handle.expect("onos>")
-            
+
             if json_format:
-                if not grep_str:
-                    self.handle.sendline("hosts -j")
-                    self.handle.expect("hosts -j")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("hosts -j | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("hosts -j | grep '"+str(grep_str)+"'")
-                    self.handle.expect("onos>")
+                self.handle.sendline("hosts -j")
+                self.handle.expect("hosts -j")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 '''
                 handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
                 To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
                 In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
-                So we take off that escape sequence using 
+                So we take off that escape sequence using
                 ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
-                handle1 = ansi_escape.sub('', handle) 
+                handle1 = ansi_escape.sub('', handle)
                 '''
                 #print "repr(handle) =", repr(handle)
                 ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -819,13 +763,8 @@
                 #print "repr(handle1) = ", repr(handle1)
                 return handle1
             else:
-                if not grep_str:
-                    self.handle.sendline("hosts")
-                    self.handle.expect("onos>")
-                else:
-                    self.handle.sendline("hosts | grep '"+
-                        str(grep_str)+"'")
-                    self.handle.expect("onos>")
+                self.handle.sendline("hosts")
+                self.handle.expect("onos>")
                 handle = self.handle.before
                 #print "handle =",handle
                 return handle
@@ -923,24 +862,24 @@
             * host_id_two: ONOS host id for host2
         Description:
             Adds a host-to-host intent (bidrectional) by
-            specifying the two hosts. 
+            specifying the two hosts.
         '''
         try:
             self.handle.sendline("")
             self.handle.expect("onos>")
-            
+
             self.handle.sendline("add-host-intent "+
                     str(host_id_one) + " " + str(host_id_two))
             self.handle.expect("onos>")
 
             handle = self.handle.before
-            print "handle =", handle
+            #print "handle =", handle
 
             main.log.info("Intent installed between "+
                     str(host_id_one) + " and " + str(host_id_two))
 
             return handle
-        
+
         except pexpect.EOF:
             main.log.error(self.name + ": EOF exception found")
             main.log.error(self.name + ":    " + self.handle.before)
@@ -1177,7 +1116,7 @@
             main.cleanup()
             main.exit()
 
-    def intents(self, json_format = False):
+    def intents(self, json_format = True):
         '''
         Optional:
             * json_format: enable output formatting in json
@@ -1215,7 +1154,7 @@
             main.cleanup()
             main.exit()
 
-    def flows(self, json_format = False):
+    def flows(self, json_format = True):
         '''
         Optional:
             * json_format: enable output formatting in json
@@ -1228,6 +1167,8 @@
                 self.handle.expect("flows -j")
                 self.handle.expect("onos>")
                 handle = self.handle.before
+                ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
+                handle = ansi_escape.sub('', handle)
 
             else:
                 self.handle.sendline("")
@@ -1235,6 +1176,8 @@
                 self.handle.sendline("flows")
                 self.handle.expect("onos>")
                 handle = self.handle.before
+            if re.search("Error\sexecuting\scommand:", handle):
+                main.log.error(self.name + ".flows() response: " + str(handle))
 
             return handle
 
@@ -1651,5 +1594,54 @@
             main.cleanup()
             main.exit()
 
+    def clusters(self, json_format=True):
+        '''
+        Lists all clusters
+        Optional argument:
+            * json_format - boolean indicating if you want output in json
+        '''
+        try:
+            self.handle.sendline("")
+            self.handle.expect("onos>")
+
+            if json_format:
+                self.handle.sendline("clusters -j")
+                self.handle.expect("clusters -j")
+                self.handle.expect("onos>")
+                handle = self.handle.before
+                '''
+                handle variable here contains some ANSI escape color code
+                sequences at the end which are invisible in the print command
+                output. To make that escape sequence visible, use repr() function.
+                The repr(handle) output when printed shows the ANSI escape sequences.
+                In json.loads(somestring), this somestring variable is actually
+                repr(somestring) and json.loads would fail with the escape sequence.
+                So we take off that escape sequence using
+                ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
+                handle1 = ansi_escape.sub('', handle)
+                '''
+                #print "repr(handle) =", repr(handle)
+                ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
+                handle1 = ansi_escape.sub('', handle)
+                #print "repr(handle1) = ", repr(handle1)
+                return handle1
+            else:
+                self.handle.sendline("clusters")
+                self.handle.expect("onos>")
+                handle = self.handle.before
+                #print "handle =",handle
+                return handle
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit()
+
 
     #***********************************
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 8489c98..e2625cf 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -82,7 +82,7 @@
         '''
         response = ''
         try:
-            self.handle.sendline("\n")
+            self.handle.sendline("")
             self.handle.expect("\$")
             self.handle.sendline("exit")
             self.handle.expect("closed")
@@ -164,7 +164,7 @@
             self.handle.sendline("cd "+ self.home)
             self.handle.expect("\$")
 
-            self.handle.sendline("\n")
+            self.handle.sendline("")
             self.handle.expect("\$")
             self.handle.sendline("mvn clean install")
             self.handle.expect("mvn clean install")
@@ -195,7 +195,7 @@
                     for line in self.handle.before.splitlines():
                         if "Total time:" in line:
                             main.log.info(line)
-                    self.handle.sendline("\n")
+                    self.handle.sendline("")
                     self.handle.expect("\$", timeout=60)
                     return main.TRUE
                 elif i == 4:
@@ -392,11 +392,11 @@
             self.handle.sendline("export TERM=xterm-256color")
             self.handle.expect("xterm-256color")
             self.handle.expect("\$")
-            self.handle.sendline("\n")
+            self.handle.sendline("")
             self.handle.expect("\$")
             self.handle.sendline("cd " + self.home + "; git log -1 --pretty=fuller --decorate=short | grep -A 6 \"commit\" --color=never")
-            #self.handle.expect("--color=never")
-            #self.handle.sendline("")
+            #NOTE: for some reason there are backspaces inserted in this phrase when run from Jenkins on some tests
+            self.handle.expect("never")
             self.handle.expect("\$")
             response=(self.name +": \n"+ str(self.handle.before + self.handle.after))
             self.handle.sendline("cd " + self.home)
@@ -411,13 +411,18 @@
                     #as xml specific tags that cause errors
                     line = line.replace("<","[")
                     line = line.replace(">","]")
-                    main.log.report(line)
+                    main.log.report("\t" + line)
             return lines[2]
         except pexpect.EOF:
             main.log.error(self.name + ": EOF exception found")
             main.log.error(self.name + ":     " + self.handle.before)
             main.cleanup()
             main.exit()
+        except pexpect.TIMEOUT:
+            main.log.error(self.name + ": TIMEOUT exception found")
+            main.log.error(self.name + ":     " + self.handle.before)
+            main.cleanup()
+            main.exit()
         except:
             main.log.info(self.name + ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::")
             main.log.error( traceback.print_exc() )
@@ -784,7 +789,7 @@
         try:
             self.handle.sendline("")
             self.handle.expect("\$")
-            self.handle.sendline("onos-uninstall "+str(node_ip))
+            self.handle.sendline( "onos-uninstall "+str(node_ip) )
             self.handle.expect("\$")
 
             main.log.info("ONOS "+node_ip+" was uninstalled")
@@ -933,14 +938,15 @@
             self.handle.sendline("onos-wait-for-start " + node )
             self.handle.expect("onos-wait-for-start")
             #NOTE: this timeout is arbitrary"
-            i = self.handle.expect(["\$", pexpect.TIMEOUT], timeout = 300)
+            i = self.handle.expect(["\$", pexpect.TIMEOUT], timeout = 120)
             if i == 0:
                 main.log.info(self.name + ": " + node + " is up")
                 return main.TRUE
             elif i == 1:
                 #NOTE: since this function won't return until ONOS is ready,
                 #   we will kill it on timeout
-                self.handle.sendline("\003")    #Control-C
+                main.log.error("ONOS has not started yet")
+                self.handle.send("\x03")    #Control-C
                 self.handle.expect("\$")
                 return main.FALSE
         except pexpect.EOF:
@@ -1238,5 +1244,3 @@
             main.log.error( traceback.print_exc())
             main.log.info(self.name+" ::::::")
 
-
-
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 9ae90fb..7bf7667 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -76,7 +76,7 @@
 	    elif i==6:
 	        main.log.info("Password not required logged in")
 
-        self.handle.sendline("\n")
+        self.handle.sendline("")
         self.handle.expect('>|#|\$')
         return self.handle
 
@@ -142,7 +142,7 @@
             main.log.info("Found More screen to go , Sending a key to proceed")
             indexMore = self.handle.expect(["^:$", expectPrompt], timeout = timeoutVar)
             while indexMore == 0:
-                main.log.info("Found anoother More screen to go , Sending a key to proceed")
+                main.log.info("Found another More screen to go , Sending a key to proceed")
                 self.handle.sendcontrol("D")
                 indexMore = self.handle.expect(["^:$", expectPrompt], timeout = timeoutVar)
                 self.LASTRSP = self.LASTRSP + self.handle.before
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
new file mode 100644
index 0000000..0a8bd57
--- /dev/null
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -0,0 +1,63 @@
+<PARAMS>
+    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <ENV>
+    <cellName>HA</cellName>
+    </ENV>
+    <Git>False</Git>
+
+    <CTRL>
+        <ip1>10.128.30.11</ip1>
+        <port1>6633</port1>
+
+        <ip2>10.128.30.12</ip2>
+        <port2>6633</port2>
+
+        <ip3>10.128.30.13</ip3>
+        <port3>6633</port3>
+
+        <ip4>10.128.30.14</ip4>
+        <port4>6633</port4>
+
+        <ip5>10.128.30.15</ip5>
+        <port5>6633</port5>
+
+        <ip6>10.128.30.16</ip6>
+        <port6>6633</port6>
+
+        <ip7>10.128.30.17</ip7>
+        <port7>6633</port7>
+    </CTRL>
+    <TESTONUSER>admin</TESTONUSER>
+    <TESTONIP>10.128.30.10</TESTONIP>
+    <PING>
+        <source1>h8</source1>
+        <source2>h9</source2>
+        <source3>h10</source3>
+        <source4>h11</source4>
+        <source5>h12</source5>
+        <source6>h13</source6>
+        <source7>h14</source7>
+        <source8>h15</source8>
+        <source9>h16</source9>
+        <source10>h17</source10>
+        <target1>10.0.0.18</target1>
+        <target2>10.0.0.19</target2>
+        <target3>10.0.0.20</target3>
+        <target4>10.0.0.21</target4>
+        <target5>10.0.0.22</target5>
+        <target6>10.0.0.23</target6>
+        <target7>10.0.0.24</target7>
+        <target8>10.0.0.25</target8>
+        <target9>10.0.0.26</target9>
+        <target10>10.0.0.27</target10>
+    </PING>
+    <timers>
+        <LinkDiscovery>2</LinkDiscovery>
+        <SwitchDiscovery>2</SwitchDiscovery>
+    </timers>
+    <MNtcpdump>
+        <intf>eth0</intf>
+        <port> </port>
+        <folder>~/packet_captures/</folder>
+    </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
new file mode 100644
index 0000000..47dc744
--- /dev/null
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -0,0 +1,1202 @@
+'''
+Description: This test is to determine if ONOS can handle
+    all of it's nodes restarting
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+'''
+class HATestClusterRestart:
+
+    def __init__(self) :
+        self.default = ''
+
+    def CASE1(self,main) :
+        '''
+        CASE1 is to compile ONOS and push it to the test machines
+
+        Startup sequence:
+        git pull
+        mvn clean install
+        onos-package
+        cell <name>
+        onos-verify-cell
+        NOTE: temporary - onos-remove-raft-logs
+        onos-install -f
+        onos-wait-for-start
+        '''
+        import time
+        main.log.report("ONOS HA test: Restart all ONOS nodes - initialization")
+        main.case("Setting up test environment")
+
+        # load some vairables from the params file
+        PULL_CODE = False
+        if main.params['Git'] == 'True':
+            PULL_CODE = True
+        cell_name = main.params['ENV']['cellName']
+
+        #set global variables
+        global ONOS1_ip
+        global ONOS1_port
+        global ONOS2_ip
+        global ONOS2_port
+        global ONOS3_ip
+        global ONOS3_port
+        global ONOS4_ip
+        global ONOS4_port
+        global ONOS5_ip
+        global ONOS5_port
+        global ONOS6_ip
+        global ONOS6_port
+        global ONOS7_ip
+        global ONOS7_port
+
+        ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS1_port = main.params['CTRL']['port1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS2_port = main.params['CTRL']['port2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS3_port = main.params['CTRL']['port3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS4_port = main.params['CTRL']['port4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS5_port = main.params['CTRL']['port5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS6_port = main.params['CTRL']['port6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+        ONOS7_port = main.params['CTRL']['port7']
+
+
+        main.step("Applying cell variable to environment")
+        cell_result = main.ONOSbench.set_cell(cell_name)
+        verify_result = main.ONOSbench.verify_cell()
+
+        #FIXME:this is short term fix 
+        main.log.report("Removing raft logs")
+        main.ONOSbench.onos_remove_raft_logs()
+        main.log.report("Uninstalling ONOS")
+        main.ONOSbench.onos_uninstall(ONOS1_ip)
+        main.ONOSbench.onos_uninstall(ONOS2_ip)
+        main.ONOSbench.onos_uninstall(ONOS3_ip)
+        main.ONOSbench.onos_uninstall(ONOS4_ip)
+        main.ONOSbench.onos_uninstall(ONOS5_ip)
+        main.ONOSbench.onos_uninstall(ONOS6_ip)
+        main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+        clean_install_result = main.TRUE
+        git_pull_result = main.TRUE
+
+        main.step("Compiling the latest version of ONOS")
+        if PULL_CODE:
+            main.step("Git checkout and pull master")
+            main.ONOSbench.git_checkout("master")
+            git_pull_result = main.ONOSbench.git_pull()
+
+            main.step("Using mvn clean & install")
+            clean_install_result = main.TRUE
+            if git_pull_result == main.TRUE:
+                clean_install_result = main.ONOSbench.clean_install()
+            else:
+                main.log.warn("Did not pull new code so skipping mvn "+ \
+                        "clean install")
+        main.ONOSbench.get_version(report=True)
+
+        main.step("Creating ONOS package")
+        package_result = main.ONOSbench.onos_package()
+
+        main.step("Installing ONOS package")
+        onos1_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS1_ip)
+        onos2_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS2_ip)
+        onos3_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS3_ip)
+        onos4_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS4_ip)
+        onos5_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS5_ip)
+        onos6_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS6_ip)
+        onos7_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS7_ip)
+        onos_install_result = onos1_install_result and onos2_install_result\
+                and onos3_install_result and onos4_install_result\
+                and onos5_install_result and onos6_install_result\
+                and onos7_install_result
+        '''
+        #FIXME: work around until onos is less fragile
+        main.ONOSbench.handle.sendline("onos-cluster-install")
+        print main.ONOSbench.handle.expect("\$")
+        onos_install_result = main.TRUE
+        '''
+
+
+        main.step("Checking if ONOS is up yet")
+        #TODO: Refactor
+        # check bundle:list?
+        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+        if not onos1_isup:
+            main.log.report("ONOS1 didn't start!")
+        onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+        if not onos2_isup:
+            main.log.report("ONOS2 didn't start!")
+        onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+        if not onos3_isup:
+            main.log.report("ONOS3 didn't start!")
+        onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+        if not onos4_isup:
+            main.log.report("ONOS4 didn't start!")
+        onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+        if not onos5_isup:
+            main.log.report("ONOS5 didn't start!")
+        onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+        if not onos6_isup:
+            main.log.report("ONOS6 didn't start!")
+        onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+        if not onos7_isup:
+            main.log.report("ONOS7 didn't start!")
+        onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+                and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+        # TODO: if it becomes an issue, we can retry this step  a few times
+
+
+        cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+        cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+        cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+        cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
+        cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
+        cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
+        cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
+        cli_results = cli_result1 and cli_result2 and cli_result3 and\
+                cli_result4 and cli_result5 and cli_result6 and cli_result7
+
+        main.step("Start Packet Capture MN")
+        main.Mininet2.start_tcpdump(
+                str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+                intf = main.params['MNtcpdump']['intf'],
+                port = main.params['MNtcpdump']['port'])
+
+
+        case1_result = (clean_install_result and package_result and
+                cell_result and verify_result and onos_install_result and
+                onos_isup_result and cli_results)
+
+        utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+                onpass="Test startup successful",
+                onfail="Test startup NOT successful")
+
+
+        #if case1_result==main.FALSE:
+        #    main.cleanup()
+        #    main.exit()
+
+    def CASE2(self,main) :
+        '''
+        Assign mastership to controllers
+        '''
+        import time
+        import json
+        import re
+
+
+        '''
+        ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS1_port = main.params['CTRL']['port1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS2_port = main.params['CTRL']['port2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS3_port = main.params['CTRL']['port3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS4_port = main.params['CTRL']['port4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS5_port = main.params['CTRL']['port5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS6_port = main.params['CTRL']['port6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+        ONOS7_port = main.params['CTRL']['port7']
+        '''
+
+
+        main.log.report("Assigning switches to controllers")
+        main.case("Assigning Controllers")
+        main.step("Assign switches to controllers")
+
+        for i in range (1,29):
+           main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+                    ip1=ONOS1_ip,port1=ONOS1_port,
+                    ip2=ONOS2_ip,port2=ONOS2_port,
+                    ip3=ONOS3_ip,port3=ONOS3_port,
+                    ip4=ONOS4_ip,port4=ONOS4_port,
+                    ip5=ONOS5_ip,port5=ONOS5_port,
+                    ip6=ONOS6_ip,port6=ONOS6_port,
+                    ip7=ONOS7_ip,port7=ONOS7_port)
+
+        mastership_check = main.TRUE
+        for i in range (1,29):
+            response = main.Mininet1.get_sw_controller("s"+str(i))
+            try:
+                main.log.info(str(response))
+            except:
+                main.log.info(repr(response))
+            if re.search("tcp:"+ONOS1_ip,response)\
+                    and re.search("tcp:"+ONOS2_ip,response)\
+                    and re.search("tcp:"+ONOS3_ip,response)\
+                    and re.search("tcp:"+ONOS4_ip,response)\
+                    and re.search("tcp:"+ONOS5_ip,response)\
+                    and re.search("tcp:"+ONOS6_ip,response)\
+                    and re.search("tcp:"+ONOS7_ip,response):
+                mastership_check = mastership_check and main.TRUE
+            else:
+                mastership_check = main.FALSE
+        if mastership_check == main.TRUE:
+            main.log.report("Switch mastership assigned correctly")
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership assigned correctly",
+                onfail="Switches not assigned correctly to controllers")
+
+        #TODO: If assign roles is working reliably then manually 
+        #   assign mastership to the controller we want
+
+
+    def CASE3(self,main) :
+        """
+        Assign intents
+
+        """
+        import time
+        import json
+        import re
+        main.log.report("Adding host intents")
+        main.case("Adding host Intents")
+
+        main.step("Discovering  Hosts( Via pingall for now)")
+        #FIXME: Once we have a host discovery mechanism, use that instead
+
+        #REACTIVE FWD test
+        ping_result = main.FALSE
+        time1 = time.time()
+        ping_result = main.Mininet1.pingall()
+        time2 = time.time()
+        main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+        #uninstall onos-app-fwd
+        main.log.info("Uninstall reactive forwarding app")
+        main.ONOScli1.feature_uninstall("onos-app-fwd")
+        main.ONOScli2.feature_uninstall("onos-app-fwd")
+        main.ONOScli3.feature_uninstall("onos-app-fwd")
+        main.ONOScli4.feature_uninstall("onos-app-fwd")
+        main.ONOScli5.feature_uninstall("onos-app-fwd")
+        main.ONOScli6.feature_uninstall("onos-app-fwd")
+        main.ONOScli7.feature_uninstall("onos-app-fwd")
+
+        main.step("Add  host intents")
+        #TODO:  move the host numbers to params
+        import json
+        intents_json= json.loads(main.ONOScli1.hosts())
+        intent_add_result = main.FALSE
+        for i in range(8,18):
+            main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+            host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+            host2 =  "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+            #NOTE: get host can return None
+            #TODO: handle this
+            host1_id = main.ONOScli1.get_host(host1)['id']
+            host2_id = main.ONOScli1.get_host(host2)['id']
+            tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+            intent_add_result = intent_add_result and tmp_result
+        #TODO Check if intents all exist in datastore
+        #NOTE: Do we need to print this once the test is working?
+        #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+        #    sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+    def CASE4(self,main) :
+        """
+        Ping across added host intents
+        """
+        description = " Ping across added host intents"
+        main.log.report(description)
+        main.case(description)
+        Ping_Result = main.TRUE
+        for i in range(8,18):
+            ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+            Ping_Result = Ping_Result and ping
+            if ping==main.FALSE:
+                main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+            elif ping==main.TRUE:
+                main.log.info("Ping test passed!")
+                Ping_Result = main.TRUE
+        if Ping_Result==main.FALSE:
+            main.log.report("Intents have not been installed correctly, pings failed.")
+        if Ping_Result==main.TRUE:
+            main.log.report("Intents have been installed correctly and verified by pings")
+        utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+                onpass="Intents have been installed correctly and pings work",
+                onfail ="Intents have not been installed correctly, pings failed." )
+
+    def CASE5(self,main) :
+        '''
+        Reading state of ONOS
+        '''
+        import time
+        import json
+        from subprocess import Popen, PIPE
+        from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+        main.log.report("Setting up and gathering data for current state")
+        main.case("Setting up and gathering data for current state")
+        #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+        #We can then compare them with eachother and also with past states
+
+        main.step("Get the Mastership of each switch from each controller")
+        global mastership_state
+        ONOS1_mastership = main.ONOScli1.roles()
+        ONOS2_mastership = main.ONOScli2.roles()
+        ONOS3_mastership = main.ONOScli3.roles()
+        ONOS4_mastership = main.ONOScli4.roles()
+        ONOS5_mastership = main.ONOScli5.roles()
+        ONOS6_mastership = main.ONOScli6.roles()
+        ONOS7_mastership = main.ONOScli7.roles()
+        #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+        if "Error" in ONOS1_mastership or not ONOS1_mastership\
+                or "Error" in ONOS2_mastership or not ONOS2_mastership\
+                or "Error" in ONOS3_mastership or not ONOS3_mastership\
+                or "Error" in ONOS4_mastership or not ONOS4_mastership\
+                or "Error" in ONOS5_mastership or not ONOS5_mastership\
+                or "Error" in ONOS6_mastership or not ONOS6_mastership\
+                or "Error" in ONOS7_mastership or not ONOS7_mastership:
+                    main.log.report("Error in getting ONOS roles")
+                    main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+                    main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+                    main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+                    main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+                    main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+                    main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+                    main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+                    consistent_mastership = main.FALSE
+        elif ONOS1_mastership == ONOS2_mastership\
+                and ONOS1_mastership == ONOS3_mastership\
+                and ONOS1_mastership == ONOS4_mastership\
+                and ONOS1_mastership == ONOS5_mastership\
+                and ONOS1_mastership == ONOS6_mastership\
+                and ONOS1_mastership == ONOS7_mastership:
+                    mastership_state = ONOS1_mastership
+                    consistent_mastership = main.TRUE
+                    main.log.report("Switch roles are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            consistent_mastership = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+                onpass="Switch roles are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of switch roles")
+
+
+        main.step("Get the intents from each controller")
+        global intent_state
+        ONOS1_intents = main.ONOScli1.intents( json_format=True )
+        ONOS2_intents = main.ONOScli2.intents( json_format=True )
+        ONOS3_intents = main.ONOScli3.intents( json_format=True )
+        ONOS4_intents = main.ONOScli4.intents( json_format=True )
+        ONOS5_intents = main.ONOScli5.intents( json_format=True )
+        ONOS6_intents = main.ONOScli6.intents( json_format=True )
+        ONOS7_intents = main.ONOScli7.intents( json_format=True )
+        intent_check = main.FALSE
+        if "Error" in ONOS1_intents or not ONOS1_intents\
+                or "Error" in ONOS2_intents or not ONOS2_intents\
+                or "Error" in ONOS3_intents or not ONOS3_intents\
+                or "Error" in ONOS4_intents or not ONOS4_intents\
+                or "Error" in ONOS5_intents or not ONOS5_intents\
+                or "Error" in ONOS6_intents or not ONOS6_intents\
+                or "Error" in ONOS7_intents or not ONOS7_intents:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+                    main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+                    main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+                    main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+                    main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+                    main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+                    main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+        elif ONOS1_intents == ONOS2_intents\
+                and ONOS1_intents == ONOS3_intents\
+                and ONOS1_intents == ONOS4_intents\
+                and ONOS1_intents == ONOS5_intents\
+                and ONOS1_intents == ONOS6_intents\
+                and ONOS1_intents == ONOS7_intents:
+                    intent_state = ONOS1_intents
+                    intent_check = main.TRUE
+                    main.log.report("Intents are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+                onpass="Intents are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of intents")
+
+
+        main.step("Get the flows from each controller")
+        global flow_state
+        ONOS1_flows = main.ONOScli1.flows( json_format=True )
+        ONOS2_flows = main.ONOScli2.flows( json_format=True )
+        ONOS3_flows = main.ONOScli3.flows( json_format=True )
+        ONOS4_flows = main.ONOScli4.flows( json_format=True )
+        ONOS5_flows = main.ONOScli5.flows( json_format=True )
+        ONOS6_flows = main.ONOScli6.flows( json_format=True )
+        ONOS7_flows = main.ONOScli7.flows( json_format=True )
+        flow_check = main.FALSE
+        if "Error" in ONOS1_flows or not ONOS1_flows\
+                or "Error" in ONOS2_flows or not ONOS2_flows\
+                or "Error" in ONOS3_flows or not ONOS3_flows\
+                or "Error" in ONOS4_flows or not ONOS4_flows\
+                or "Error" in ONOS5_flows or not ONOS5_flows\
+                or "Error" in ONOS6_flows or not ONOS6_flows\
+                or "Error" in ONOS7_flows or not ONOS7_flows:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+                    main.log.warn("ONOS2 flows repsponse: "+ ONOS2_flows)
+                    main.log.warn("ONOS3 flows repsponse: "+ ONOS3_flows)
+                    main.log.warn("ONOS4 flows repsponse: "+ ONOS4_flows)
+                    main.log.warn("ONOS5 flows repsponse: "+ ONOS5_flows)
+                    main.log.warn("ONOS6 flows repsponse: "+ ONOS6_flows)
+                    main.log.warn("ONOS7 flows repsponse: "+ ONOS7_flows)
+        elif len(json.loads(ONOS1_flows)) == len(json.loads(ONOS2_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS3_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS4_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS5_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS6_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS7_flows)):
+                #TODO: Do a better check, maybe compare flows on switches?
+                    flow_state = ONOS1_flows
+                    flow_check = main.TRUE
+                    main.log.report("Flow count is consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 flows: "+ json.dumps(json.loads(ONOS1_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 flows: "+ json.dumps(json.loads(ONOS2_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 flows: "+ json.dumps(json.loads(ONOS3_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 flows: "+ json.dumps(json.loads(ONOS4_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 flows: "+ json.dumps(json.loads(ONOS5_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 flows: "+ json.dumps(json.loads(ONOS6_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 flows: "+ json.dumps(json.loads(ONOS7_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=flow_check,
+                onpass="The flow count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different flow counts")
+
+
+        main.step("Get the OF Table entries")
+        global flows
+        flows=[]
+        for i in range(1,29):
+            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+
+        #TODO: Compare switch flow tables with ONOS flow tables
+
+        main.step("Start continuous pings")
+        main.Mininet2.pingLong(src=main.params['PING']['source1'],
+                            target=main.params['PING']['target1'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source2'],
+                            target=main.params['PING']['target2'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source3'],
+                            target=main.params['PING']['target3'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source4'],
+                            target=main.params['PING']['target4'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source5'],
+                            target=main.params['PING']['target5'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source6'],
+                            target=main.params['PING']['target6'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source7'],
+                            target=main.params['PING']['target7'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source8'],
+                            target=main.params['PING']['target8'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source9'],
+                            target=main.params['PING']['target9'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source10'],
+                            target=main.params['PING']['target10'],pingTime=500)
+
+        main.step("Create TestONTopology object")
+        ctrls = []
+        count = 1
+        while True:
+            temp = ()
+            if ('ip' + str(count)) in main.params['CTRL']:
+                temp = temp + (getattr(main,('ONOS' + str(count))),)
+                temp = temp + ("ONOS"+str(count),)
+                temp = temp + (main.params['CTRL']['ip'+str(count)],)
+                temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+                ctrls.append(temp)
+                count = count + 1
+            else:
+                break
+        MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+        main.step("Collecting topology information from ONOS")
+        devices = []
+        devices.append( main.ONOScli1.devices() )
+        devices.append( main.ONOScli2.devices() )
+        devices.append( main.ONOScli3.devices() )
+        devices.append( main.ONOScli4.devices() )
+        devices.append( main.ONOScli5.devices() )
+        devices.append( main.ONOScli6.devices() )
+        devices.append( main.ONOScli7.devices() )
+        '''
+        hosts = []
+        hosts.append( main.ONOScli1.hosts() )
+        hosts.append( main.ONOScli2.hosts() )
+        hosts.append( main.ONOScli3.hosts() )
+        hosts.append( main.ONOScli4.hosts() )
+        hosts.append( main.ONOScli5.hosts() )
+        hosts.append( main.ONOScli6.hosts() )
+        hosts.append( main.ONOScli7.hosts() )
+        '''
+        ports = []
+        ports.append( main.ONOScli1.ports() )
+        ports.append( main.ONOScli2.ports() )
+        ports.append( main.ONOScli3.ports() )
+        ports.append( main.ONOScli4.ports() )
+        ports.append( main.ONOScli5.ports() )
+        ports.append( main.ONOScli6.ports() )
+        ports.append( main.ONOScli7.ports() )
+        links = []
+        links.append( main.ONOScli1.links() )
+        links.append( main.ONOScli2.links() )
+        links.append( main.ONOScli3.links() )
+        links.append( main.ONOScli4.links() )
+        links.append( main.ONOScli5.links() )
+        links.append( main.ONOScli6.links() )
+        links.append( main.ONOScli7.links() )
+
+
+        main.step("Comparing ONOS topology to MN")
+        devices_results = main.TRUE
+        ports_results = main.TRUE
+        links_results = main.TRUE
+        for controller in range(7): #TODO parameterize the number of controllers
+            if devices[controller] or not "Error" in devices[controller]:
+                current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+            else:
+                current_devices_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                    onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+            if ports[controller] or not "Error" in ports[controller]:
+                current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+            else:
+                current_ports_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                    onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+            if links[controller] or not "Error" in links[controller]:
+                current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+            else:
+                current_links_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                    onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+            devices_results = devices_results and current_devices_result
+            ports_results = ports_results and current_ports_result
+            links_results = links_results and current_links_result
+
+        topo_result = devices_results and ports_results and links_results
+        utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+                onpass="Topology Check Test successful",
+                onfail="Topology Check Test NOT successful")
+
+        final_assert = main.TRUE
+        final_assert = final_assert and topo_result and flow_check \
+                and intent_check and consistent_mastership
+        utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+                onpass="State check successful",
+                onfail="State check NOT successful")
+
+
+    def CASE6(self,main) :
+        '''
+        The Failure case.
+        '''
+        main.log.report("Restart entire ONOS cluster")
+        main.log.case("Restart entire ONOS cluster")
+        main.ONOSbench.onos_kill(ONOS1_ip)
+        main.ONOSbench.onos_kill(ONOS2_ip)
+        main.ONOSbench.onos_kill(ONOS3_ip)
+        main.ONOSbench.onos_kill(ONOS4_ip)
+        main.ONOSbench.onos_kill(ONOS5_ip)
+        main.ONOSbench.onos_kill(ONOS6_ip)
+        main.ONOSbench.onos_kill(ONOS7_ip)
+
+        main.step("Checking if ONOS is up yet")
+        count = 0
+        onos_isup_result = main.FALSE
+        while onos_isup_result == main.FALSE and count < 10:
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+            onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+            onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+            onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+            onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+            onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+            onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+                    and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+            count = count + 1
+        # TODO: if it becomes an issue, we can retry this step  a few times
+
+
+        cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+        cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+        cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+        cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
+        cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
+        cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
+        cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
+        cli_results = cli_result1 and cli_result2 and cli_result3\
+                and cli_result4 and cli_result5 and cli_result6\
+                and cli_result7
+
+        case_results = main.TRUE and onos_isup_result and cli_results
+        utilities.assert_equals(expect=main.TRUE, actual=case_results,
+                onpass="ONOS restart successful",
+                onfail="ONOS restart NOT successful")
+
+
+    def CASE7(self,main) :
+        '''
+        Check state after ONOS failure
+        '''
+        import os
+        import json
+        main.case("Running ONOS Constant State Tests")
+
+        main.step("Check if switch roles are consistent across all nodes")
+        ONOS1_mastership = main.ONOScli1.roles()
+        ONOS2_mastership = main.ONOScli2.roles()
+        ONOS3_mastership = main.ONOScli3.roles()
+        ONOS4_mastership = main.ONOScli4.roles()
+        ONOS5_mastership = main.ONOScli5.roles()
+        ONOS6_mastership = main.ONOScli6.roles()
+        ONOS7_mastership = main.ONOScli7.roles()
+        #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+        if "Error" in ONOS1_mastership or not ONOS1_mastership\
+                or "Error" in ONOS2_mastership or not ONOS2_mastership\
+                or "Error" in ONOS3_mastership or not ONOS3_mastership\
+                or "Error" in ONOS4_mastership or not ONOS4_mastership\
+                or "Error" in ONOS5_mastership or not ONOS5_mastership\
+                or "Error" in ONOS6_mastership or not ONOS6_mastership\
+                or "Error" in ONOS7_mastership or not ONOS7_mastership:
+                    main.log.error("Error in getting ONOS mastership")
+                    main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+                    main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+                    main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+                    main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+                    main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+                    main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+                    main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+                    consistent_mastership = main.FALSE
+        elif ONOS1_mastership == ONOS2_mastership\
+                and ONOS1_mastership == ONOS3_mastership\
+                and ONOS1_mastership == ONOS4_mastership\
+                and ONOS1_mastership == ONOS5_mastership\
+                and ONOS1_mastership == ONOS6_mastership\
+                and ONOS1_mastership == ONOS7_mastership:
+                    #mastership_state = ONOS1_mastership
+                    consistent_mastership = main.TRUE
+                    main.log.report("Switch roles are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            consistent_mastership = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+                onpass="Switch roles are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of switch roles")
+
+
+        description2 = "Compare switch roles from before failure"
+        main.step(description2)
+
+        current_json = json.loads(ONOS1_mastership)
+        old_json = json.loads(mastership_state)
+        mastership_check = main.TRUE
+        for i in range(1,29):
+            switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+            current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+            old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+            if current == old:
+                mastership_check = mastership_check and main.TRUE
+            else:
+                main.log.warn("Mastership of switch %s changed" % switchDPID)
+                mastership_check = main.FALSE
+        if mastership_check == main.TRUE:
+            main.log.report("Mastership of Switches was not changed")
+        utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+                onpass="Mastership of Switches was not changed",
+                onfail="Mastership of some switches changed")
+        #NOTE: we expect mastership to change on controller failure
+        mastership_check = mastership_check #and consistent_mastership
+
+
+
+        main.step("Get the intents and compare across all nodes")
+        ONOS1_intents = main.ONOScli1.intents( json_format=True )
+        ONOS2_intents = main.ONOScli2.intents( json_format=True )
+        ONOS3_intents = main.ONOScli3.intents( json_format=True )
+        ONOS4_intents = main.ONOScli4.intents( json_format=True )
+        ONOS5_intents = main.ONOScli5.intents( json_format=True )
+        ONOS6_intents = main.ONOScli6.intents( json_format=True )
+        ONOS7_intents = main.ONOScli7.intents( json_format=True )
+        intent_check = main.FALSE
+        if "Error" in ONOS1_intents or not ONOS1_intents\
+                or "Error" in ONOS2_intents or not ONOS2_intents\
+                or "Error" in ONOS3_intents or not ONOS3_intents\
+                or "Error" in ONOS4_intents or not ONOS4_intents\
+                or "Error" in ONOS5_intents or not ONOS5_intents\
+                or "Error" in ONOS6_intents or not ONOS6_intents\
+                or "Error" in ONOS7_intents or not ONOS7_intents:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+                    main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+                    main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+                    main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+                    main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+                    main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+                    main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+        elif ONOS1_intents == ONOS2_intents\
+                and ONOS1_intents == ONOS3_intents\
+                and ONOS1_intents == ONOS4_intents\
+                and ONOS1_intents == ONOS5_intents\
+                and ONOS1_intents == ONOS6_intents\
+                and ONOS1_intents == ONOS7_intents:
+                    intent_check = main.TRUE
+                    main.log.report("Intents are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+                onpass="Intents are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of intents")
+
+        main.step("Compare current intents with intents before the failure")
+        if intent_state == ONOS1_intents:
+            same_intents = main.TRUE
+            main.log.report("Intents are consistent with before failure")
+        #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+        else:
+            same_intents = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+                onpass="Intents are consistent with before failure",
+                onfail="The Intents changed during failure")
+        intent_check = intent_check and same_intents
+
+
+
+        main.step("Get the OF Table entries and compare to before component failure")
+        Flow_Tables = main.TRUE
+        flows2=[]
+        for i in range(28):
+            main.log.info("Checking flow table on s" + str(i+1))
+            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            flows2.append(tmp_flows)
+            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            if Flow_Tables == main.FALSE:
+                main.log.info("Differences in flow table for switch: "+str(i+1))
+                break
+        if Flow_Tables == main.TRUE:
+            main.log.report("No changes were found in the flow tables")
+        utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+                onpass="No changes were found in the flow tables",
+                onfail="Changes were found in the flow tables")
+
+        main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+        #FIXME: This check is always failing. Investigate cause
+        #NOTE:  this may be something to do with file permsissions
+        #       or slight change in format
+        main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
+        Loss_In_Pings = main.FALSE 
+        #NOTE: checkForLoss returns main.FALSE with 0% packet loss
+        for i in range(8,18):
+            main.log.info("Checking for a loss in pings along flow from s" + str(i))
+            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+        if Loss_In_Pings == main.TRUE:
+            main.log.info("Loss in ping detected")
+        elif Loss_In_Pings == main.ERROR:
+            main.log.info("There are multiple mininet process running")
+        elif Loss_In_Pings == main.FALSE:
+            main.log.info("No Loss in the pings")
+            main.log.report("No loss of dataplane connectivity")
+        utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
+                onpass="No Loss of connectivity",
+                onfail="Loss of dataplane connectivity detected")
+
+
+        #TODO:add topology to this or leave as a seperate case?
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = int(result)
+        if result == main.TRUE:
+            main.log.report("Constant State Tests Passed")
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Constant State Tests Passed",
+                onfail="Constant state tests failed")
+
+    def CASE8 (self,main):
+        '''
+        Compare topo
+        '''
+        import sys
+        sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+        from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+        import json
+        import time
+
+        description ="Compare ONOS Topology view to Mininet topology"
+        main.case(description)
+        main.log.report(description)
+        main.step("Create TestONTopology object")
+        ctrls = []
+        count = 1
+        while True:
+            temp = ()
+            if ('ip' + str(count)) in main.params['CTRL']:
+                temp = temp + (getattr(main,('ONOS' + str(count))),)
+                temp = temp + ("ONOS"+str(count),)
+                temp = temp + (main.params['CTRL']['ip'+str(count)],)
+                temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+                ctrls.append(temp)
+                count = count + 1
+            else:
+                break
+        MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+        main.step("Comparing ONOS topology to MN")
+        devices_results = main.TRUE
+        ports_results = main.TRUE
+        links_results = main.TRUE
+        topo_result = main.FALSE
+        start_time = time.time()
+        elapsed = 0
+        count = 0
+        while topo_result == main.FALSE and elapsed < 120:
+            count = count + 1
+            try:
+                main.step("Collecting topology information from ONOS")
+                devices = []
+                devices.append( main.ONOScli1.devices() )
+                devices.append( main.ONOScli2.devices() )
+                devices.append( main.ONOScli3.devices() )
+                devices.append( main.ONOScli4.devices() )
+                devices.append( main.ONOScli5.devices() )
+                devices.append( main.ONOScli6.devices() )
+                devices.append( main.ONOScli7.devices() )
+                '''
+                hosts = []
+                hosts.append( main.ONOScli1.hosts() )
+                hosts.append( main.ONOScli2.hosts() )
+                hosts.append( main.ONOScli3.hosts() )
+                hosts.append( main.ONOScli4.hosts() )
+                hosts.append( main.ONOScli5.hosts() )
+                hosts.append( main.ONOScli6.hosts() )
+                hosts.append( main.ONOScli7.hosts() )
+                '''
+                ports = []
+                ports.append( main.ONOScli1.ports() )
+                ports.append( main.ONOScli2.ports() )
+                ports.append( main.ONOScli3.ports() )
+                ports.append( main.ONOScli4.ports() )
+                ports.append( main.ONOScli5.ports() )
+                ports.append( main.ONOScli6.ports() )
+                ports.append( main.ONOScli7.ports() )
+                links = []
+                links.append( main.ONOScli1.links() )
+                links.append( main.ONOScli2.links() )
+                links.append( main.ONOScli3.links() )
+                links.append( main.ONOScli4.links() )
+                links.append( main.ONOScli5.links() )
+                links.append( main.ONOScli6.links() )
+                links.append( main.ONOScli7.links() )
+
+                for controller in range(7): #TODO parameterize the number of controllers
+                    if devices[controller] or not "Error" in devices[controller]:
+                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                    else:
+                        current_devices_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+                    if ports[controller] or not "Error" in ports[controller]:
+                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                    else:
+                        current_ports_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                    if links[controller] or not "Error" in links[controller]:
+                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                    else:
+                        current_links_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+            except:
+                main.log.error("something went wrong in topo comparison")
+                main.log.warn( repr( devices ) )
+                main.log.warn( repr( ports ) )
+                main.log.warn( repr( links ) )
+
+            devices_results = devices_results and current_devices_result
+            ports_results = ports_results and current_ports_result
+            links_results = links_results and current_links_result
+            topo_result = devices_results and ports_results and links_results
+            elapsed = time.time() - start_time
+        time_threshold = elapsed < 1
+        topo_result = topo_result and time_threshold
+        #TODO make sure this step is non-blocking. IE add a timeout
+        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+                str(elapsed) + " seconds, " + str(count) +" tries" )
+        utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+                onpass="Topology Check Test successful",
+                onfail="Topology Check Test NOT successful")
+        if topo_result == main.TRUE:
+            main.log.report("ONOS topology view matches Mininet topology")
+
+
+    def CASE9 (self,main):
+        '''
+        Link s3-s28 down
+        '''
+        #NOTE: You should probably run a topology check after this
+
+        link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+        description = "Turn off a link to ensure that Link Discovery is working properly"
+        main.log.report(description)
+        main.case(description)
+
+
+        main.step("Kill Link between s3 and s28")
+        Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+        main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+        time.sleep(link_sleep)
+        utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+                onpass="Link down succesful",
+                onfail="Failed to bring link down")
+        #TODO do some sort of check here
+
+    def CASE10 (self,main):
+        '''
+        Link s3-s28 up
+        '''
+        #NOTE: You should probably run a topology check after this
+
+        link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+        description = "Restore a link to ensure that Link Discovery is working properly"
+        main.log.report(description)
+        main.case(description)
+
+        main.step("Bring link between s3 and s28 back up")
+        Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+        main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+        time.sleep(link_sleep)
+        utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+                onpass="Link up succesful",
+                onfail="Failed to bring link up")
+        #TODO do some sort of check here
+
+
+    def CASE11 (self, main) :
+        '''
+        Switch Down
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time
+
+        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+        description = "Killing a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        #TODO: Make this switch parameterizable
+        main.step("Kill s28 ")
+        main.log.report("Deleting s28")
+        #FIXME: use new dynamic topo functions
+        main.Mininet1.del_switch("s28")
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the deleted switch
+        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+        #TODO do some sort of check here
+
+    def CASE12 (self, main) :
+        '''
+        Switch Up
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time
+        #FIXME: use new dynamic topo functions
+        description = "Adding a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        main.step("Add back s28")
+        main.log.report("Adding back s28")
+        main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+        #TODO: New dpid or same? Ask Thomas?
+        main.Mininet1.add_link('s28', 's3')
+        main.Mininet1.add_link('s28', 's6')
+        main.Mininet1.add_link('s28', 'h28')
+        main.Mininet1.assign_sw_controller(sw="28",count=7,
+                ip1=ONOS1_ip,port1=ONOS1_port,
+                ip2=ONOS2_ip,port2=ONOS2_port,
+                ip3=ONOS3_ip,port3=ONOS3_port,
+                ip4=ONOS4_ip,port4=ONOS4_port,
+                ip5=ONOS5_ip,port5=ONOS5_port,
+                ip6=ONOS6_ip,port6=ONOS6_port,
+                ip7=ONOS7_ip,port7=ONOS7_port)
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the added switch
+        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+        #TODO do some sort of check here
+
+    def CASE13 (self, main) :
+        '''
+        Clean up
+        '''
+        import os
+        import time
+        description = "Test Cleanup"
+        main.log.report(description)
+        main.case(description)
+        main.step("Killing tcpdumps")
+        main.Mininet2.stop_tcpdump()
+
+        main.step("Copying MN pcap and ONOS log files to test station")
+        testname = main.TEST
+        #NOTE: MN Pcap file is being saved to ~/packet_captures
+        #       scp this file as MN and TestON aren't necessarily the same vm
+        #FIXME: scp
+        #####mn files
+        #TODO: Load these from params
+        #NOTE: must end in /
+        log_folder = "/opt/onos/log/"
+        log_files = ["karaf.log", "karaf.log.1"]
+        #NOTE: must end in /
+        dst_dir = "~/packet_captures/"
+        for f in log_files:
+            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS1-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS2-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS3-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS4-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS5-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS6-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS7-"+f )
+
+        #std*.log's
+        #NOTE: must end in /
+        log_folder = "/opt/onos/var/"
+        log_files = ["stderr.log", "stdout.log"]
+        #NOTE: must end in /
+        dst_dir = "~/packet_captures/"
+        for f in log_files:
+            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS1-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS2-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS3-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS4-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS5-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS6-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS7-"+f )
+
+
+
+
+        #sleep so scp can finish
+        time.sleep(10)
+        main.step("Packing and rotating pcap archives")
+        os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+        #TODO: actually check something here
+        utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+                onpass="Test cleanup successful",
+                onfail="Test cleanup NOT successful")
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.topo b/TestON/tests/HATestClusterRestart/HATestClusterRestart.topo
new file mode 100644
index 0000000..4d4156c
--- /dev/null
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.topo
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli1>
+
+        <ONOScli2>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli2>
+
+        <ONOScli3>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli3>
+
+
+        <ONOScli4>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli4>
+
+
+        <ONOScli5>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli5>
+
+
+        <ONOScli6>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli6>
+
+
+        <ONOScli7>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>8</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli7>
+
+        <ONOS1>
+            <host>10.128.30.11</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>9</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS1>
+
+        <ONOS2>
+            <host>10.128.30.12</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>10</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS2>
+
+        <ONOS3>
+            <host>10.128.30.13</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>11</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS3>
+
+        <ONOS4>
+            <host>10.128.30.14</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>12</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS4>
+
+        <ONOS5>
+            <host>10.128.30.15</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>13</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS5>
+
+        <ONOS6>
+            <host>10.128.30.16</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>14</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS6>
+
+        <ONOS7>
+            <host>10.128.30.17</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>15</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS7>
+
+        <Mininet1>
+            <host>10.128.30.9</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>MininetCliDriver</type>
+            <connect_order>16</connect_order>
+            <COMPONENTS>
+                #Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+                <arg2> --topo mytopo </arg2>
+                <arg3> </arg3>
+                <controller> remote </controller>
+            </COMPONENTS>
+        </Mininet1>
+
+        <Mininet2>
+            <host>10.128.30.9</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>RemoteMininetDriver</type>
+            <connect_order>17</connect_order>
+            <COMPONENTS>
+                # Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+                <arg2> --topo mytopo --arp</arg2>
+                <controller> remote </controller>
+             </COMPONENTS>
+        </Mininet2>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestClusterRestart/__init__.py b/TestON/tests/HATestClusterRestart/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HATestClusterRestart/__init__.py
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
new file mode 100644
index 0000000..0a8bd57
--- /dev/null
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -0,0 +1,63 @@
+<PARAMS>
+    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <ENV>
+    <cellName>HA</cellName>
+    </ENV>
+    <Git>False</Git>
+
+    <CTRL>
+        <ip1>10.128.30.11</ip1>
+        <port1>6633</port1>
+
+        <ip2>10.128.30.12</ip2>
+        <port2>6633</port2>
+
+        <ip3>10.128.30.13</ip3>
+        <port3>6633</port3>
+
+        <ip4>10.128.30.14</ip4>
+        <port4>6633</port4>
+
+        <ip5>10.128.30.15</ip5>
+        <port5>6633</port5>
+
+        <ip6>10.128.30.16</ip6>
+        <port6>6633</port6>
+
+        <ip7>10.128.30.17</ip7>
+        <port7>6633</port7>
+    </CTRL>
+    <TESTONUSER>admin</TESTONUSER>
+    <TESTONIP>10.128.30.10</TESTONIP>
+    <PING>
+        <source1>h8</source1>
+        <source2>h9</source2>
+        <source3>h10</source3>
+        <source4>h11</source4>
+        <source5>h12</source5>
+        <source6>h13</source6>
+        <source7>h14</source7>
+        <source8>h15</source8>
+        <source9>h16</source9>
+        <source10>h17</source10>
+        <target1>10.0.0.18</target1>
+        <target2>10.0.0.19</target2>
+        <target3>10.0.0.20</target3>
+        <target4>10.0.0.21</target4>
+        <target5>10.0.0.22</target5>
+        <target6>10.0.0.23</target6>
+        <target7>10.0.0.24</target7>
+        <target8>10.0.0.25</target8>
+        <target9>10.0.0.26</target9>
+        <target10>10.0.0.27</target10>
+    </PING>
+    <timers>
+        <LinkDiscovery>2</LinkDiscovery>
+        <SwitchDiscovery>2</SwitchDiscovery>
+    </timers>
+    <MNtcpdump>
+        <intf>eth0</intf>
+        <port> </port>
+        <folder>~/packet_captures/</folder>
+    </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
new file mode 100644
index 0000000..2af775d
--- /dev/null
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -0,0 +1,1202 @@
+'''
+Description: This test is to determine if ONOS can handle
+    a minority of it's nodes restarting
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+'''
+class HATestMinorityRestart:
+
+    def __init__(self) :
+        self.default = ''
+
+    def CASE1(self,main) :
+        '''
+        CASE1 is to compile ONOS and push it to the test machines
+
+        Startup sequence:
+        git pull
+        mvn clean install
+        onos-package
+        cell <name>
+        onos-verify-cell
+        NOTE: temporary - onos-remove-raft-logs
+        onos-install -f
+        onos-wait-for-start
+        '''
+        import time
+        main.log.report("ONOS HA test: Restart minority of ONOS nodes - initialization")
+        main.case("Setting up test environment")
+
+        # load some vairables from the params file
+        PULL_CODE = False
+        if main.params['Git'] == 'True':
+            PULL_CODE = True
+        cell_name = main.params['ENV']['cellName']
+
+        #set global variables
+        global ONOS1_ip
+        global ONOS1_port
+        global ONOS2_ip
+        global ONOS2_port
+        global ONOS3_ip
+        global ONOS3_port
+        global ONOS4_ip
+        global ONOS4_port
+        global ONOS5_ip
+        global ONOS5_port
+        global ONOS6_ip
+        global ONOS6_port
+        global ONOS7_ip
+        global ONOS7_port
+
+        ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS1_port = main.params['CTRL']['port1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS2_port = main.params['CTRL']['port2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS3_port = main.params['CTRL']['port3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS4_port = main.params['CTRL']['port4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS5_port = main.params['CTRL']['port5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS6_port = main.params['CTRL']['port6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+        ONOS7_port = main.params['CTRL']['port7']
+
+
+        main.step("Applying cell variable to environment")
+        cell_result = main.ONOSbench.set_cell(cell_name)
+        verify_result = main.ONOSbench.verify_cell()
+
+        #FIXME:this is short term fix 
+        main.log.report("Removing raft logs")
+        main.ONOSbench.onos_remove_raft_logs()
+        main.log.report("Uninstalling ONOS")
+        main.ONOSbench.onos_uninstall(ONOS1_ip)
+        main.ONOSbench.onos_uninstall(ONOS2_ip)
+        main.ONOSbench.onos_uninstall(ONOS3_ip)
+        main.ONOSbench.onos_uninstall(ONOS4_ip)
+        main.ONOSbench.onos_uninstall(ONOS5_ip)
+        main.ONOSbench.onos_uninstall(ONOS6_ip)
+        main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+        clean_install_result = main.TRUE
+        git_pull_result = main.TRUE
+
+        main.step("Compiling the latest version of ONOS")
+        if PULL_CODE:
+            main.step("Git checkout and pull master")
+            main.ONOSbench.git_checkout("master")
+            git_pull_result = main.ONOSbench.git_pull()
+
+            main.step("Using mvn clean & install")
+            clean_install_result = main.TRUE
+            if git_pull_result == main.TRUE:
+                clean_install_result = main.ONOSbench.clean_install()
+            else:
+                main.log.warn("Did not pull new code so skipping mvn "+ \
+                        "clean install")
+        main.ONOSbench.get_version(report=True)
+
+        main.step("Creating ONOS package")
+        package_result = main.ONOSbench.onos_package()
+
+        main.step("Installing ONOS package")
+        onos1_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS1_ip)
+        onos2_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS2_ip)
+        onos3_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS3_ip)
+        onos4_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS4_ip)
+        onos5_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS5_ip)
+        onos6_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS6_ip)
+        onos7_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS7_ip)
+        onos_install_result = onos1_install_result and onos2_install_result\
+                and onos3_install_result and onos4_install_result\
+                and onos5_install_result and onos6_install_result\
+                and onos7_install_result
+        '''
+        #FIXME: work around until onos is less fragile
+        main.ONOSbench.handle.sendline("onos-cluster-install")
+        print main.ONOSbench.handle.expect("\$")
+        onos_install_result = main.TRUE
+        '''
+
+
+        main.step("Checking if ONOS is up yet")
+        #TODO: Refactor
+        # check bundle:list?
+        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+        if not onos1_isup:
+            main.log.report("ONOS1 didn't start!")
+        onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+        if not onos2_isup:
+            main.log.report("ONOS2 didn't start!")
+        onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+        if not onos3_isup:
+            main.log.report("ONOS3 didn't start!")
+        onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+        if not onos4_isup:
+            main.log.report("ONOS4 didn't start!")
+        onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+        if not onos5_isup:
+            main.log.report("ONOS5 didn't start!")
+        onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+        if not onos6_isup:
+            main.log.report("ONOS6 didn't start!")
+        onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+        if not onos7_isup:
+            main.log.report("ONOS7 didn't start!")
+        onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+                and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+        # TODO: if it becomes an issue, we can retry this step  a few times
+
+
+        cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+        cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+        cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+        cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
+        cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
+        cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
+        cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
+        cli_results = cli_result1 and cli_result2 and cli_result3 and\
+                cli_result4 and cli_result5 and cli_result6 and cli_result7
+
+        main.step("Start Packet Capture MN")
+        main.Mininet2.start_tcpdump(
+                str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+                intf = main.params['MNtcpdump']['intf'],
+                port = main.params['MNtcpdump']['port'])
+
+
+        case1_result = (clean_install_result and package_result and
+                cell_result and verify_result and onos_install_result and
+                onos_isup_result and cli_results)
+
+        utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+                onpass="Test startup successful",
+                onfail="Test startup NOT successful")
+
+
+        #if case1_result==main.FALSE:
+        #    main.cleanup()
+        #    main.exit()
+
+    def CASE2(self,main) :
+        '''
+        Assign mastership to controllers
+        '''
+        import time
+        import json
+        import re
+
+
+        '''
+        ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS1_port = main.params['CTRL']['port1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS2_port = main.params['CTRL']['port2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS3_port = main.params['CTRL']['port3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS4_port = main.params['CTRL']['port4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS5_port = main.params['CTRL']['port5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS6_port = main.params['CTRL']['port6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+        ONOS7_port = main.params['CTRL']['port7']
+        '''
+
+
+        main.log.report("Assigning switches to controllers")
+        main.case("Assigning Controllers")
+        main.step("Assign switches to controllers")
+
+        for i in range (1,29):
+           main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+                    ip1=ONOS1_ip,port1=ONOS1_port,
+                    ip2=ONOS2_ip,port2=ONOS2_port,
+                    ip3=ONOS3_ip,port3=ONOS3_port,
+                    ip4=ONOS4_ip,port4=ONOS4_port,
+                    ip5=ONOS5_ip,port5=ONOS5_port,
+                    ip6=ONOS6_ip,port6=ONOS6_port,
+                    ip7=ONOS7_ip,port7=ONOS7_port)
+
+        mastership_check = main.TRUE
+        for i in range (1,29):
+            response = main.Mininet1.get_sw_controller("s"+str(i))
+            try:
+                main.log.info(str(response))
+            except:
+                main.log.info(repr(response))
+            if re.search("tcp:"+ONOS1_ip,response)\
+                    and re.search("tcp:"+ONOS2_ip,response)\
+                    and re.search("tcp:"+ONOS3_ip,response)\
+                    and re.search("tcp:"+ONOS4_ip,response)\
+                    and re.search("tcp:"+ONOS5_ip,response)\
+                    and re.search("tcp:"+ONOS6_ip,response)\
+                    and re.search("tcp:"+ONOS7_ip,response):
+                mastership_check = mastership_check and main.TRUE
+            else:
+                mastership_check = main.FALSE
+        if mastership_check == main.TRUE:
+            main.log.report("Switch mastership assigned correctly")
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership assigned correctly",
+                onfail="Switches not assigned correctly to controllers")
+
+        #TODO: If assign roles is working reliably then manually 
+        #   assign mastership to the controller we want
+
+
+    def CASE3(self,main) :
+        """
+        Assign intents
+
+        """
+        import time
+        import json
+        import re
+        main.log.report("Adding host intents")
+        main.case("Adding host Intents")
+
+        main.step("Discovering  Hosts( Via pingall for now)")
+        #FIXME: Once we have a host discovery mechanism, use that instead
+
+        #REACTIVE FWD test
+        ping_result = main.FALSE
+        time1 = time.time()
+        ping_result = main.Mininet1.pingall()
+        time2 = time.time()
+        main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+        #uninstall onos-app-fwd
+        main.log.info("Uninstall reactive forwarding app")
+        main.ONOScli1.feature_uninstall("onos-app-fwd")
+        main.ONOScli2.feature_uninstall("onos-app-fwd")
+        main.ONOScli3.feature_uninstall("onos-app-fwd")
+        main.ONOScli4.feature_uninstall("onos-app-fwd")
+        main.ONOScli5.feature_uninstall("onos-app-fwd")
+        main.ONOScli6.feature_uninstall("onos-app-fwd")
+        main.ONOScli7.feature_uninstall("onos-app-fwd")
+
+        main.step("Add  host intents")
+        #TODO:  move the host numbers to params
+        import json
+        intents_json= json.loads(main.ONOScli1.hosts())
+        intent_add_result = main.FALSE
+        for i in range(8,18):
+            main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+            host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+            host2 =  "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+            #NOTE: get host can return None
+            #TODO: handle this
+            host1_id = main.ONOScli1.get_host(host1)['id']
+            host2_id = main.ONOScli1.get_host(host2)['id']
+            tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+            intent_add_result = intent_add_result and tmp_result
+        #TODO Check if intents all exist in datastore
+        #NOTE: Do we need to print this once the test is working?
+        #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+        #    sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+    def CASE4(self,main) :
+        """
+        Ping across added host intents
+        """
+        description = " Ping across added host intents"
+        main.log.report(description)
+        main.case(description)
+        Ping_Result = main.TRUE
+        for i in range(8,18):
+            ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+            Ping_Result = Ping_Result and ping
+            if ping==main.FALSE:
+                main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+            elif ping==main.TRUE:
+                main.log.info("Ping test passed!")
+                Ping_Result = main.TRUE
+        if Ping_Result==main.FALSE:
+            main.log.report("Intents have not been installed correctly, pings failed.")
+        if Ping_Result==main.TRUE:
+            main.log.report("Intents have been installed correctly and verified by pings")
+        utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+                onpass="Intents have been installed correctly and pings work",
+                onfail ="Intents have not been installed correctly, pings failed." )
+
+    def CASE5(self,main) :
+        '''
+        Reading state of ONOS
+        '''
+        import time
+        import json
+        from subprocess import Popen, PIPE
+        from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+        main.log.report("Setting up and gathering data for current state")
+        main.case("Setting up and gathering data for current state")
+        #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+        #We can then compare them with eachother and also with past states
+
+        main.step("Get the Mastership of each switch from each controller")
+        global mastership_state
+        ONOS1_mastership = main.ONOScli1.roles()
+        ONOS2_mastership = main.ONOScli2.roles()
+        ONOS3_mastership = main.ONOScli3.roles()
+        ONOS4_mastership = main.ONOScli4.roles()
+        ONOS5_mastership = main.ONOScli5.roles()
+        ONOS6_mastership = main.ONOScli6.roles()
+        ONOS7_mastership = main.ONOScli7.roles()
+        #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+        if "Error" in ONOS1_mastership or not ONOS1_mastership\
+                or "Error" in ONOS2_mastership or not ONOS2_mastership\
+                or "Error" in ONOS3_mastership or not ONOS3_mastership\
+                or "Error" in ONOS4_mastership or not ONOS4_mastership\
+                or "Error" in ONOS5_mastership or not ONOS5_mastership\
+                or "Error" in ONOS6_mastership or not ONOS6_mastership\
+                or "Error" in ONOS7_mastership or not ONOS7_mastership:
+                    main.log.report("Error in getting ONOS roles")
+                    main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+                    main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+                    main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+                    main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+                    main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+                    main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+                    main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+                    consistent_mastership = main.FALSE
+        elif ONOS1_mastership == ONOS2_mastership\
+                and ONOS1_mastership == ONOS3_mastership\
+                and ONOS1_mastership == ONOS4_mastership\
+                and ONOS1_mastership == ONOS5_mastership\
+                and ONOS1_mastership == ONOS6_mastership\
+                and ONOS1_mastership == ONOS7_mastership:
+                    mastership_state = ONOS1_mastership
+                    consistent_mastership = main.TRUE
+                    main.log.report("Switch roles are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            consistent_mastership = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+                onpass="Switch roles are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of switch roles")
+
+
+        main.step("Get the intents from each controller")
+        global intent_state
+        ONOS1_intents = main.ONOScli1.intents( json_format=True )
+        ONOS2_intents = main.ONOScli2.intents( json_format=True )
+        ONOS3_intents = main.ONOScli3.intents( json_format=True )
+        ONOS4_intents = main.ONOScli4.intents( json_format=True )
+        ONOS5_intents = main.ONOScli5.intents( json_format=True )
+        ONOS6_intents = main.ONOScli6.intents( json_format=True )
+        ONOS7_intents = main.ONOScli7.intents( json_format=True )
+        intent_check = main.FALSE
+        if "Error" in ONOS1_intents or not ONOS1_intents\
+                or "Error" in ONOS2_intents or not ONOS2_intents\
+                or "Error" in ONOS3_intents or not ONOS3_intents\
+                or "Error" in ONOS4_intents or not ONOS4_intents\
+                or "Error" in ONOS5_intents or not ONOS5_intents\
+                or "Error" in ONOS6_intents or not ONOS6_intents\
+                or "Error" in ONOS7_intents or not ONOS7_intents:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+                    main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+                    main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+                    main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+                    main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+                    main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+                    main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+        elif ONOS1_intents == ONOS2_intents\
+                and ONOS1_intents == ONOS3_intents\
+                and ONOS1_intents == ONOS4_intents\
+                and ONOS1_intents == ONOS5_intents\
+                and ONOS1_intents == ONOS6_intents\
+                and ONOS1_intents == ONOS7_intents:
+                    intent_state = ONOS1_intents
+                    intent_check = main.TRUE
+                    main.log.report("Intents are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+                onpass="Intents are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of intents")
+
+
+        main.step("Get the flows from each controller")
+        global flow_state
+        ONOS1_flows = main.ONOScli1.flows( json_format=True )
+        ONOS2_flows = main.ONOScli2.flows( json_format=True )
+        ONOS3_flows = main.ONOScli3.flows( json_format=True )
+        ONOS4_flows = main.ONOScli4.flows( json_format=True )
+        ONOS5_flows = main.ONOScli5.flows( json_format=True )
+        ONOS6_flows = main.ONOScli6.flows( json_format=True )
+        ONOS7_flows = main.ONOScli7.flows( json_format=True )
+        flow_check = main.FALSE
+        if "Error" in ONOS1_flows or not ONOS1_flows\
+                or "Error" in ONOS2_flows or not ONOS2_flows\
+                or "Error" in ONOS3_flows or not ONOS3_flows\
+                or "Error" in ONOS4_flows or not ONOS4_flows\
+                or "Error" in ONOS5_flows or not ONOS5_flows\
+                or "Error" in ONOS6_flows or not ONOS6_flows\
+                or "Error" in ONOS7_flows or not ONOS7_flows:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+                    main.log.warn("ONOS2 flows repsponse: "+ ONOS2_flows)
+                    main.log.warn("ONOS3 flows repsponse: "+ ONOS3_flows)
+                    main.log.warn("ONOS4 flows repsponse: "+ ONOS4_flows)
+                    main.log.warn("ONOS5 flows repsponse: "+ ONOS5_flows)
+                    main.log.warn("ONOS6 flows repsponse: "+ ONOS6_flows)
+                    main.log.warn("ONOS7 flows repsponse: "+ ONOS7_flows)
+        elif len(json.loads(ONOS1_flows)) == len(json.loads(ONOS2_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS3_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS4_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS5_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS6_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS7_flows)):
+                #TODO: Do a better check, maybe compare flows on switches?
+                    flow_state = ONOS1_flows
+                    flow_check = main.TRUE
+                    main.log.report("Flow count is consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 flows: "+ json.dumps(json.loads(ONOS1_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 flows: "+ json.dumps(json.loads(ONOS2_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 flows: "+ json.dumps(json.loads(ONOS3_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 flows: "+ json.dumps(json.loads(ONOS4_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 flows: "+ json.dumps(json.loads(ONOS5_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 flows: "+ json.dumps(json.loads(ONOS6_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 flows: "+ json.dumps(json.loads(ONOS7_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=flow_check,
+                onpass="The flow count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different flow counts")
+
+
+        main.step("Get the OF Table entries")
+        global flows
+        flows=[]
+        for i in range(1,29):
+            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+
+        #TODO: Compare switch flow tables with ONOS flow tables
+
+        main.step("Start continuous pings")
+        main.Mininet2.pingLong(src=main.params['PING']['source1'],
+                            target=main.params['PING']['target1'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source2'],
+                            target=main.params['PING']['target2'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source3'],
+                            target=main.params['PING']['target3'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source4'],
+                            target=main.params['PING']['target4'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source5'],
+                            target=main.params['PING']['target5'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source6'],
+                            target=main.params['PING']['target6'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source7'],
+                            target=main.params['PING']['target7'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source8'],
+                            target=main.params['PING']['target8'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source9'],
+                            target=main.params['PING']['target9'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source10'],
+                            target=main.params['PING']['target10'],pingTime=500)
+
+        main.step("Create TestONTopology object")
+        ctrls = []
+        count = 1
+        while True:
+            temp = ()
+            if ('ip' + str(count)) in main.params['CTRL']:
+                temp = temp + (getattr(main,('ONOS' + str(count))),)
+                temp = temp + ("ONOS"+str(count),)
+                temp = temp + (main.params['CTRL']['ip'+str(count)],)
+                temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+                ctrls.append(temp)
+                count = count + 1
+            else:
+                break
+        MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+        main.step("Collecting topology information from ONOS")
+        devices = []
+        devices.append( main.ONOScli1.devices() )
+        devices.append( main.ONOScli2.devices() )
+        devices.append( main.ONOScli3.devices() )
+        devices.append( main.ONOScli4.devices() )
+        devices.append( main.ONOScli5.devices() )
+        devices.append( main.ONOScli6.devices() )
+        devices.append( main.ONOScli7.devices() )
+        '''
+        hosts = []
+        hosts.append( main.ONOScli1.hosts() )
+        hosts.append( main.ONOScli2.hosts() )
+        hosts.append( main.ONOScli3.hosts() )
+        hosts.append( main.ONOScli4.hosts() )
+        hosts.append( main.ONOScli5.hosts() )
+        hosts.append( main.ONOScli6.hosts() )
+        hosts.append( main.ONOScli7.hosts() )
+        '''
+        ports = []
+        ports.append( main.ONOScli1.ports() )
+        ports.append( main.ONOScli2.ports() )
+        ports.append( main.ONOScli3.ports() )
+        ports.append( main.ONOScli4.ports() )
+        ports.append( main.ONOScli5.ports() )
+        ports.append( main.ONOScli6.ports() )
+        ports.append( main.ONOScli7.ports() )
+        links = []
+        links.append( main.ONOScli1.links() )
+        links.append( main.ONOScli2.links() )
+        links.append( main.ONOScli3.links() )
+        links.append( main.ONOScli4.links() )
+        links.append( main.ONOScli5.links() )
+        links.append( main.ONOScli6.links() )
+        links.append( main.ONOScli7.links() )
+
+
+        main.step("Comparing ONOS topology to MN")
+        devices_results = main.TRUE
+        ports_results = main.TRUE
+        links_results = main.TRUE
+        for controller in range(7): #TODO parameterize the number of controllers
+            if devices[controller] or not "Error" in devices[controller]:
+                current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+            else:
+                current_devices_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                    onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+            if ports[controller] or not "Error" in ports[controller]:
+                current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+            else:
+                current_ports_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                    onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+            if links[controller] or not "Error" in links[controller]:
+                current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+            else:
+                current_links_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                    onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+            devices_results = devices_results and current_devices_result
+            ports_results = ports_results and current_ports_result
+            links_results = links_results and current_links_result
+
+        topo_result = devices_results and ports_results and links_results
+        utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+                onpass="Topology Check Test successful",
+                onfail="Topology Check Test NOT successful")
+
+        final_assert = main.TRUE
+        final_assert = final_assert and topo_result and flow_check \
+                and intent_check and consistent_mastership
+        utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+                onpass="State check successful",
+                onfail="State check NOT successful")
+
+
+    def CASE6(self,main) :
+        '''
+        The Failure case.
+        '''
+        main.log.report("Killing 3 ONOS nodes")
+        main.log.case("Restart minority of ONOS nodes")
+        #TODO: Randomize these nodes
+        main.ONOSbench.onos_kill(ONOS1_ip)
+        main.ONOSbench.onos_kill(ONOS2_ip)
+        main.ONOSbench.onos_kill(ONOS3_ip)
+
+        main.step("Checking if ONOS is up yet")
+        count = 0
+        onos_isup_result = main.FALSE
+        while onos_isup_result == main.FALSE and count < 10:
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+            onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+            onos_isup_result = onos1_isup and onos2_isup and onos3_isup
+            count = count + 1
+        # TODO: if it becomes an issue, we can retry this step  a few times
+
+
+        cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+        cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+        cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+        cli_results = cli_result1 and cli_result2 and cli_result3
+
+        case_results = main.TRUE and onos_isup_result and cli_results
+        utilities.assert_equals(expect=main.TRUE, actual=case_results,
+                onpass="ONOS restart successful",
+                onfail="ONOS restart NOT successful")
+
+
+    def CASE7(self,main) :
+        '''
+        Check state after ONOS failure
+        '''
+        import os
+        import json
+        main.case("Running ONOS Constant State Tests")
+
+        main.step("Check if switch roles are consistent across all nodes")
+        ONOS1_mastership = main.ONOScli1.roles()
+        ONOS2_mastership = main.ONOScli2.roles()
+        ONOS3_mastership = main.ONOScli3.roles()
+        ONOS4_mastership = main.ONOScli4.roles()
+        ONOS5_mastership = main.ONOScli5.roles()
+        ONOS6_mastership = main.ONOScli6.roles()
+        ONOS7_mastership = main.ONOScli7.roles()
+        print type(ONOS1_mastership)
+        print ONOS1_mastership
+        print type(ONOS2_mastership)
+        print ONOS2_mastership
+        print type(ONOS3_mastership)
+        print ONOS3_mastership
+        print type(ONOS4_mastership)
+        print ONOS4_mastership
+        print type(ONOS5_mastership)
+        print ONOS5_mastership
+        print type(ONOS6_mastership)
+        print ONOS6_mastership
+        print type(ONOS7_mastership)
+        print ONOS7_mastership
+        #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+        if "Error" in ONOS1_mastership or not ONOS1_mastership\
+                or "Error" in ONOS2_mastership or not ONOS2_mastership\
+                or "Error" in ONOS3_mastership or not ONOS3_mastership\
+                or "Error" in ONOS4_mastership or not ONOS4_mastership\
+                or "Error" in ONOS5_mastership or not ONOS5_mastership\
+                or "Error" in ONOS6_mastership or not ONOS6_mastership\
+                or "Error" in ONOS7_mastership or not ONOS7_mastership:
+                    main.log.error("Error in getting ONOS mastership")
+                    main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+                    main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+                    main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+                    main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+                    main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+                    main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+                    main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+                    consistent_mastership = main.FALSE
+        elif ONOS1_mastership == ONOS2_mastership\
+                and ONOS1_mastership == ONOS3_mastership\
+                and ONOS1_mastership == ONOS4_mastership\
+                and ONOS1_mastership == ONOS5_mastership\
+                and ONOS1_mastership == ONOS6_mastership\
+                and ONOS1_mastership == ONOS7_mastership:
+                    #mastership_state = ONOS1_mastership
+                    consistent_mastership = main.TRUE
+                    main.log.report("Switch roles are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            consistent_mastership = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+                onpass="Switch roles are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of switch roles")
+
+
+        description2 = "Compare switch roles from before failure"
+        main.step(description2)
+
+        current_json = json.loads(ONOS1_mastership)
+        old_json = json.loads(mastership_state)
+        mastership_check = main.TRUE
+        for i in range(1,29):
+            switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+            current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+            old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+            if current == old:
+                mastership_check = mastership_check and main.TRUE
+            else:
+                main.log.warn("Mastership of switch %s changed" % switchDPID)
+                mastership_check = main.FALSE
+        if mastership_check == main.TRUE:
+            main.log.report("Mastership of Switches was not changed")
+        utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+                onpass="Mastership of Switches was not changed",
+                onfail="Mastership of some switches changed")
+        #NOTE: we expect mastership to change on controller failure
+        mastership_check = mastership_check #and consistent_mastership
+
+
+
+        main.step("Get the intents and compare across all nodes")
+        ONOS1_intents = main.ONOScli1.intents( json_format=True )
+        ONOS2_intents = main.ONOScli2.intents( json_format=True )
+        ONOS3_intents = main.ONOScli3.intents( json_format=True )
+        ONOS4_intents = main.ONOScli4.intents( json_format=True )
+        ONOS5_intents = main.ONOScli5.intents( json_format=True )
+        ONOS6_intents = main.ONOScli6.intents( json_format=True )
+        ONOS7_intents = main.ONOScli7.intents( json_format=True )
+        intent_check = main.FALSE
+        if "Error" in ONOS1_intents or not ONOS1_intents\
+                or "Error" in ONOS2_intents or not ONOS2_intents\
+                or "Error" in ONOS3_intents or not ONOS3_intents\
+                or "Error" in ONOS4_intents or not ONOS4_intents\
+                or "Error" in ONOS5_intents or not ONOS5_intents\
+                or "Error" in ONOS6_intents or not ONOS6_intents\
+                or "Error" in ONOS7_intents or not ONOS7_intents:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+                    main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+                    main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+                    main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+                    main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+                    main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+                    main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+        elif ONOS1_intents == ONOS2_intents\
+                and ONOS1_intents == ONOS3_intents\
+                and ONOS1_intents == ONOS4_intents\
+                and ONOS1_intents == ONOS5_intents\
+                and ONOS1_intents == ONOS6_intents\
+                and ONOS1_intents == ONOS7_intents:
+                    intent_check = main.TRUE
+                    main.log.report("Intents are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+                onpass="Intents are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of intents")
+
+        main.step("Compare current intents with intents before the failure")
+        if intent_state == ONOS1_intents:
+            same_intents = main.TRUE
+            main.log.report("Intents are consistent with before failure")
+        #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+        else:
+            same_intents = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+                onpass="Intents are consistent with before failure",
+                onfail="The Intents changed during failure")
+        intent_check = intent_check and same_intents
+
+
+
+        main.step("Get the OF Table entries and compare to before component failure")
+        Flow_Tables = main.TRUE
+        flows2=[]
+        for i in range(28):
+            main.log.info("Checking flow table on s" + str(i+1))
+            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            flows2.append(tmp_flows)
+            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            if Flow_Tables == main.FALSE:
+                main.log.info("Differences in flow table for switch: "+str(i+1))
+                break
+        if Flow_Tables == main.TRUE:
+            main.log.report("No changes were found in the flow tables")
+        utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+                onpass="No changes were found in the flow tables",
+                onfail="Changes were found in the flow tables")
+
+        main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+        #FIXME: This check is always failing. Investigate cause
+        #NOTE:  this may be something to do with file permsissions
+        #       or slight change in format
+        main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
+        Loss_In_Pings = main.FALSE 
+        #NOTE: checkForLoss returns main.FALSE with 0% packet loss
+        for i in range(8,18):
+            main.log.info("Checking for a loss in pings along flow from s" + str(i))
+            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+        if Loss_In_Pings == main.TRUE:
+            main.log.info("Loss in ping detected")
+        elif Loss_In_Pings == main.ERROR:
+            main.log.info("There are multiple mininet process running")
+        elif Loss_In_Pings == main.FALSE:
+            main.log.info("No Loss in the pings")
+            main.log.report("No loss of dataplane connectivity")
+        utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
+                onpass="No Loss of connectivity",
+                onfail="Loss of dataplane connectivity detected")
+
+
+        #TODO:add topology to this or leave as a seperate case?
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = int(result)
+        if result == main.TRUE:
+            main.log.report("Constant State Tests Passed")
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Constant State Tests Passed",
+                onfail="Constant state tests failed")
+
+    def CASE8 (self,main):
+        '''
+        Compare topo
+        '''
+        import sys
+        sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+        from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+        import json
+        import time
+
+        description ="Compare ONOS Topology view to Mininet topology"
+        main.case(description)
+        main.log.report(description)
+        main.step("Create TestONTopology object")
+        ctrls = []
+        count = 1
+        while True:
+            temp = ()
+            if ('ip' + str(count)) in main.params['CTRL']:
+                temp = temp + (getattr(main,('ONOS' + str(count))),)
+                temp = temp + ("ONOS"+str(count),)
+                temp = temp + (main.params['CTRL']['ip'+str(count)],)
+                temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+                ctrls.append(temp)
+                count = count + 1
+            else:
+                break
+        MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+        main.step("Comparing ONOS topology to MN")
+        devices_results = main.TRUE
+        ports_results = main.TRUE
+        links_results = main.TRUE
+        topo_result = main.FALSE
+        start_time = time.time()
+        elapsed = 0
+        count = 0
+        while topo_result == main.FALSE and elapsed < 120:
+            count = count + 1
+            try:
+                main.step("Collecting topology information from ONOS")
+                devices = []
+                devices.append( main.ONOScli1.devices() )
+                devices.append( main.ONOScli2.devices() )
+                devices.append( main.ONOScli3.devices() )
+                devices.append( main.ONOScli4.devices() )
+                devices.append( main.ONOScli5.devices() )
+                devices.append( main.ONOScli6.devices() )
+                devices.append( main.ONOScli7.devices() )
+                '''
+                hosts = []
+                hosts.append( main.ONOScli1.hosts() )
+                hosts.append( main.ONOScli2.hosts() )
+                hosts.append( main.ONOScli3.hosts() )
+                hosts.append( main.ONOScli4.hosts() )
+                hosts.append( main.ONOScli5.hosts() )
+                hosts.append( main.ONOScli6.hosts() )
+                hosts.append( main.ONOScli7.hosts() )
+                '''
+                ports = []
+                ports.append( main.ONOScli1.ports() )
+                ports.append( main.ONOScli2.ports() )
+                ports.append( main.ONOScli3.ports() )
+                ports.append( main.ONOScli4.ports() )
+                ports.append( main.ONOScli5.ports() )
+                ports.append( main.ONOScli6.ports() )
+                ports.append( main.ONOScli7.ports() )
+                links = []
+                links.append( main.ONOScli1.links() )
+                links.append( main.ONOScli2.links() )
+                links.append( main.ONOScli3.links() )
+                links.append( main.ONOScli4.links() )
+                links.append( main.ONOScli5.links() )
+                links.append( main.ONOScli6.links() )
+                links.append( main.ONOScli7.links() )
+
+                for controller in range(7): #TODO parameterize the number of controllers
+                    if devices[controller] or not "Error" in devices[controller]:
+                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                    else:
+                        current_devices_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+                    if ports[controller] or not "Error" in ports[controller]:
+                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                    else:
+                        current_ports_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                    if links[controller] or not "Error" in links[controller]:
+                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                    else:
+                        current_links_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+            except:
+                main.log.error("something went wrong in topo comparison")
+                main.log.warn( repr( devices ) )
+                main.log.warn( repr( ports ) )
+                main.log.warn( repr( links ) )
+
+            devices_results = devices_results and current_devices_result
+            ports_results = ports_results and current_ports_result
+            links_results = links_results and current_links_result
+            topo_result = devices_results and ports_results and links_results
+            elapsed = time.time() - start_time
+        time_threshold = elapsed < 1
+        topo_result = topo_result and time_threshold
+        #TODO make sure this step is non-blocking. IE add a timeout
+        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+                str(elapsed) + " seconds, " + str(count) +" tries" )
+        utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+                onpass="Topology Check Test successful",
+                onfail="Topology Check Test NOT successful")
+        if topo_result == main.TRUE:
+            main.log.report("ONOS topology view matches Mininet topology")
+
+
+    def CASE9 (self,main):
+        '''
+        Link s3-s28 down
+        '''
+        #NOTE: You should probably run a topology check after this
+
+        link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+        description = "Turn off a link to ensure that Link Discovery is working properly"
+        main.log.report(description)
+        main.case(description)
+
+
+        main.step("Kill Link between s3 and s28")
+        Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+        main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+        time.sleep(link_sleep)
+        utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+                onpass="Link down succesful",
+                onfail="Failed to bring link down")
+        #TODO do some sort of check here
+
+    def CASE10 (self,main):
+        '''
+        Link s3-s28 up
+        '''
+        #NOTE: You should probably run a topology check after this
+
+        link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+        description = "Restore a link to ensure that Link Discovery is working properly"
+        main.log.report(description)
+        main.case(description)
+
+        main.step("Bring link between s3 and s28 back up")
+        Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+        main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+        time.sleep(link_sleep)
+        utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+                onpass="Link up succesful",
+                onfail="Failed to bring link up")
+        #TODO do some sort of check here
+
+
+    def CASE11 (self, main) :
+        '''
+        Switch Down
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time
+
+        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+        description = "Killing a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        #TODO: Make this switch parameterizable
+        main.step("Kill s28 ")
+        main.log.report("Deleting s28")
+        #FIXME: use new dynamic topo functions
+        main.Mininet1.del_switch("s28")
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the deleted switch
+        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+        #TODO do some sort of check here
+
+    def CASE12 (self, main) :
+        '''
+        Switch Up
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time
+        #FIXME: use new dynamic topo functions
+        description = "Adding a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        main.step("Add back s28")
+        main.log.report("Adding back s28")
+        main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+        #TODO: New dpid or same? Ask Thomas?
+        main.Mininet1.add_link('s28', 's3')
+        main.Mininet1.add_link('s28', 's6')
+        main.Mininet1.add_link('s28', 'h28')
+        main.Mininet1.assign_sw_controller(sw="28",count=7,
+                ip1=ONOS1_ip,port1=ONOS1_port,
+                ip2=ONOS2_ip,port2=ONOS2_port,
+                ip3=ONOS3_ip,port3=ONOS3_port,
+                ip4=ONOS4_ip,port4=ONOS4_port,
+                ip5=ONOS5_ip,port5=ONOS5_port,
+                ip6=ONOS6_ip,port6=ONOS6_port,
+                ip7=ONOS7_ip,port7=ONOS7_port)
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the added switch
+        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+        #TODO do some sort of check here
+
+    def CASE13 (self, main) :
+        '''
+        Clean up
+        '''
+        import os
+        import time
+        description = "Test Cleanup"
+        main.log.report(description)
+        main.case(description)
+        main.step("Killing tcpdumps")
+        main.Mininet2.stop_tcpdump()
+
+        main.step("Copying MN pcap and ONOS log files to test station")
+        testname = main.TEST
+        #NOTE: MN Pcap file is being saved to ~/packet_captures
+        #       scp this file as MN and TestON aren't necessarily the same vm
+        #FIXME: scp
+        #####mn files
+        #TODO: Load these from params
+        #NOTE: must end in /
+        log_folder = "/opt/onos/log/"
+        log_files = ["karaf.log", "karaf.log.1"]
+        #NOTE: must end in /
+        dst_dir = "~/packet_captures/"
+        for f in log_files:
+            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS1-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS2-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS3-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS4-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS5-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS6-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS7-"+f )
+
+        #std*.log's
+        #NOTE: must end in /
+        log_folder = "/opt/onos/var/"
+        log_files = ["stderr.log", "stdout.log"]
+        #NOTE: must end in /
+        dst_dir = "~/packet_captures/"
+        for f in log_files:
+            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS1-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS2-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS3-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS4-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS5-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS6-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS7-"+f )
+
+
+
+
+        #sleep so scp can finish
+        time.sleep(10)
+        main.step("Packing and rotating pcap archives")
+        os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+        #TODO: actually check something here
+        utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+                onpass="Test cleanup successful",
+                onfail="Test cleanup NOT successful")
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.topo b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.topo
new file mode 100644
index 0000000..4d4156c
--- /dev/null
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.topo
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli1>
+
+        <ONOScli2>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli2>
+
+        <ONOScli3>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli3>
+
+
+        <ONOScli4>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli4>
+
+
+        <ONOScli5>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli5>
+
+
+        <ONOScli6>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli6>
+
+
+        <ONOScli7>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>8</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli7>
+
+        <ONOS1>
+            <host>10.128.30.11</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>9</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS1>
+
+        <ONOS2>
+            <host>10.128.30.12</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>10</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS2>
+
+        <ONOS3>
+            <host>10.128.30.13</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>11</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS3>
+
+        <ONOS4>
+            <host>10.128.30.14</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>12</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS4>
+
+        <ONOS5>
+            <host>10.128.30.15</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>13</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS5>
+
+        <ONOS6>
+            <host>10.128.30.16</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>14</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS6>
+
+        <ONOS7>
+            <host>10.128.30.17</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>15</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS7>
+
+        <Mininet1>
+            <host>10.128.30.9</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>MininetCliDriver</type>
+            <connect_order>16</connect_order>
+            <COMPONENTS>
+                #Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+                <arg2> --topo mytopo </arg2>
+                <arg3> </arg3>
+                <controller> remote </controller>
+            </COMPONENTS>
+        </Mininet1>
+
+        <Mininet2>
+            <host>10.128.30.9</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>RemoteMininetDriver</type>
+            <connect_order>17</connect_order>
+            <COMPONENTS>
+                # Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+                <arg2> --topo mytopo --arp</arg2>
+                <controller> remote </controller>
+             </COMPONENTS>
+        </Mininet2>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestMinorityRestart/__init__.py b/TestON/tests/HATestMinorityRestart/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HATestMinorityRestart/__init__.py
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
new file mode 100644
index 0000000..e1e75f9
--- /dev/null
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -0,0 +1,63 @@
+<PARAMS>
+    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <ENV>
+    <cellName>HA</cellName>
+    </ENV>
+    <Git>True</Git>
+
+    <CTRL>
+        <ip1>10.128.30.11</ip1>
+        <port1>6633</port1>
+
+        <ip2>10.128.30.12</ip2>
+        <port2>6633</port2>
+
+        <ip3>10.128.30.13</ip3>
+        <port3>6633</port3>
+
+        <ip4>10.128.30.14</ip4>
+        <port4>6633</port4>
+
+        <ip5>10.128.30.15</ip5>
+        <port5>6633</port5>
+
+        <ip6>10.128.30.16</ip6>
+        <port6>6633</port6>
+
+        <ip7>10.128.30.17</ip7>
+        <port7>6633</port7>
+    </CTRL>
+    <TESTONUSER>admin</TESTONUSER>
+    <TESTONIP>10.128.30.10</TESTONIP>
+    <PING>
+        <source1>h8</source1>
+        <source2>h9</source2>
+        <source3>h10</source3>
+        <source4>h11</source4>
+        <source5>h12</source5>
+        <source6>h13</source6>
+        <source7>h14</source7>
+        <source8>h15</source8>
+        <source9>h16</source9>
+        <source10>h17</source10>
+        <target1>10.0.0.18</target1>
+        <target2>10.0.0.19</target2>
+        <target3>10.0.0.20</target3>
+        <target4>10.0.0.21</target4>
+        <target5>10.0.0.22</target5>
+        <target6>10.0.0.23</target6>
+        <target7>10.0.0.24</target7>
+        <target8>10.0.0.25</target8>
+        <target9>10.0.0.26</target9>
+        <target10>10.0.0.27</target10>
+    </PING>
+    <timers>
+        <LinkDiscovery>2</LinkDiscovery>
+        <SwitchDiscovery>2</SwitchDiscovery>
+    </timers>
+    <MNtcpdump>
+        <intf>eth0</intf>
+        <port> </port>
+        <folder>~/packet_captures/</folder>
+    </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
new file mode 100644
index 0000000..614d514
--- /dev/null
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -0,0 +1,1171 @@
+'''
+Description: This test is to determine if the HA test setup is
+    working correctly. There are no failures so this test should
+    have a 100% pass rate
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case. Since this is the Sanity test, we do nothing.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+'''
+class HATestSanity:
+
+    def __init__(self) :
+        self.default = ''
+
+    def CASE1(self,main) :
+        '''
+        CASE1 is to compile ONOS and push it to the test machines
+
+        Startup sequence:
+        git pull
+        mvn clean install
+        onos-package
+        cell <name>
+        onos-verify-cell
+        NOTE: temporary - onos-remove-raft-logs
+        onos-install -f
+        onos-wait-for-start
+        '''
+        import time
+        main.log.report("ONOS HA Sanity test - initialization")
+        main.case("Setting up test environment")
+        #TODO: save all the timers and output them for plotting
+
+        # load some vairables from the params file
+        PULL_CODE = False
+        if main.params['Git'] == 'True':
+            PULL_CODE = True
+        cell_name = main.params['ENV']['cellName']
+
+        #set global variables
+        global ONOS1_ip
+        global ONOS1_port
+        global ONOS2_ip
+        global ONOS2_port
+        global ONOS3_ip
+        global ONOS3_port
+        global ONOS4_ip
+        global ONOS4_port
+        global ONOS5_ip
+        global ONOS5_port
+        global ONOS6_ip
+        global ONOS6_port
+        global ONOS7_ip
+        global ONOS7_port
+
+        ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS1_port = main.params['CTRL']['port1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS2_port = main.params['CTRL']['port2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS3_port = main.params['CTRL']['port3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS4_port = main.params['CTRL']['port4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS5_port = main.params['CTRL']['port5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS6_port = main.params['CTRL']['port6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+        ONOS7_port = main.params['CTRL']['port7']
+
+
+        main.step("Applying cell variable to environment")
+        cell_result = main.ONOSbench.set_cell(cell_name)
+        verify_result = main.ONOSbench.verify_cell()
+
+        #FIXME:this is short term fix
+        main.log.report("Removing raft logs")
+        main.ONOSbench.onos_remove_raft_logs()
+        main.log.report("Uninstalling ONOS")
+        main.ONOSbench.onos_uninstall(ONOS1_ip)
+        main.ONOSbench.onos_uninstall(ONOS2_ip)
+        main.ONOSbench.onos_uninstall(ONOS3_ip)
+        main.ONOSbench.onos_uninstall(ONOS4_ip)
+        main.ONOSbench.onos_uninstall(ONOS5_ip)
+        main.ONOSbench.onos_uninstall(ONOS6_ip)
+        main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+        clean_install_result = main.TRUE
+        git_pull_result = main.TRUE
+
+        main.step("Compiling the latest version of ONOS")
+        if PULL_CODE:
+            main.step("Git checkout and pull master")
+            main.ONOSbench.git_checkout("master")
+            git_pull_result = main.ONOSbench.git_pull()
+
+            main.step("Using mvn clean & install")
+            clean_install_result = main.TRUE
+            if git_pull_result == main.TRUE:
+                clean_install_result = main.ONOSbench.clean_install()
+            else:
+                main.log.warn("Did not pull new code so skipping mvn "+ \
+                        "clean install")
+        main.ONOSbench.get_version(report=True)
+
+        main.step("Creating ONOS package")
+        package_result = main.ONOSbench.onos_package()
+
+        main.step("Installing ONOS package")
+        onos1_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS1_ip)
+        onos2_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS2_ip)
+        onos3_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS3_ip)
+        onos4_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS4_ip)
+        onos5_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS5_ip)
+        onos6_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS6_ip)
+        onos7_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS7_ip)
+        onos_install_result = onos1_install_result and onos2_install_result\
+                and onos3_install_result and onos4_install_result\
+                and onos5_install_result and onos6_install_result\
+                and onos7_install_result
+        '''
+        #FIXME: work around until onos is less fragile
+        main.ONOSbench.handle.sendline("onos-cluster-install")
+        print main.ONOSbench.handle.expect("\$")
+        onos_install_result = main.TRUE
+        '''
+
+
+        main.step("Checking if ONOS is up yet")
+        #TODO: Refactor
+        # check bundle:list?
+        for i in range(2):
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            if not onos1_isup:
+                main.log.report("ONOS1 didn't start!")
+            onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+            if not onos2_isup:
+                main.log.report("ONOS2 didn't start!")
+            onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+            if not onos3_isup:
+                main.log.report("ONOS3 didn't start!")
+            onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+            if not onos4_isup:
+                main.log.report("ONOS4 didn't start!")
+            onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+            if not onos5_isup:
+                main.log.report("ONOS5 didn't start!")
+            onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+            if not onos6_isup:
+                main.log.report("ONOS6 didn't start!")
+            onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+            if not onos7_isup:
+                main.log.report("ONOS7 didn't start!")
+            onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+                    and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+            if onos_isup_result == main.TRUE:
+                break
+        # TODO: if it becomes an issue, we can retry this step  a few times
+
+
+        cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+        cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+        cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+        cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
+        cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
+        cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
+        cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
+        cli_results = cli_result1 and cli_result2 and cli_result3 and\
+                cli_result4 and cli_result5 and cli_result6 and cli_result7
+
+        main.step("Start Packet Capture MN")
+        main.Mininet2.start_tcpdump(
+                str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+                intf = main.params['MNtcpdump']['intf'],
+                port = main.params['MNtcpdump']['port'])
+
+
+        case1_result = (clean_install_result and package_result and
+                cell_result and verify_result and onos_install_result and
+                onos_isup_result and cli_results)
+
+        utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+                onpass="Test startup successful",
+                onfail="Test startup NOT successful")
+
+
+        if case1_result==main.FALSE:
+            main.cleanup()
+            main.exit()
+
+    def CASE2(self,main) :
+        '''
+        Assign mastership to controllers
+        '''
+        import time
+        import json
+        import re
+
+
+        '''
+        ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS1_port = main.params['CTRL']['port1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS2_port = main.params['CTRL']['port2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS3_port = main.params['CTRL']['port3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS4_port = main.params['CTRL']['port4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS5_port = main.params['CTRL']['port5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS6_port = main.params['CTRL']['port6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+        ONOS7_port = main.params['CTRL']['port7']
+        '''
+
+
+        main.log.report("Assigning switches to controllers")
+        main.case("Assigning Controllers")
+        main.step("Assign switches to controllers")
+
+        for i in range (1,29):
+           main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+                    ip1=ONOS1_ip,port1=ONOS1_port,
+                    ip2=ONOS2_ip,port2=ONOS2_port,
+                    ip3=ONOS3_ip,port3=ONOS3_port,
+                    ip4=ONOS4_ip,port4=ONOS4_port,
+                    ip5=ONOS5_ip,port5=ONOS5_port,
+                    ip6=ONOS6_ip,port6=ONOS6_port,
+                    ip7=ONOS7_ip,port7=ONOS7_port)
+
+        mastership_check = main.TRUE
+        for i in range (1,29):
+            response = main.Mininet1.get_sw_controller("s"+str(i))
+            try:
+                main.log.info(str(response))
+            except:
+                main.log.info(repr(response))
+            if re.search("tcp:"+ONOS1_ip,response)\
+                    and re.search("tcp:"+ONOS2_ip,response)\
+                    and re.search("tcp:"+ONOS3_ip,response)\
+                    and re.search("tcp:"+ONOS4_ip,response)\
+                    and re.search("tcp:"+ONOS5_ip,response)\
+                    and re.search("tcp:"+ONOS6_ip,response)\
+                    and re.search("tcp:"+ONOS7_ip,response):
+                mastership_check = mastership_check and main.TRUE
+            else:
+                mastership_check = main.FALSE
+        if mastership_check == main.TRUE:
+            main.log.report("Switch mastership assigned correctly")
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership assigned correctly",
+                onfail="Switches not assigned correctly to controllers")
+
+        #TODO: If assign roles is working reliably then manually 
+        #   assign mastership to the controller we want
+
+
+    def CASE3(self,main) :
+        """
+        Assign intents
+
+        """
+        import time
+        import json
+        import re
+        main.log.report("Adding host intents")
+        main.case("Adding host Intents")
+
+        main.step("Discovering  Hosts( Via pingall for now)")
+        #FIXME: Once we have a host discovery mechanism, use that instead
+
+        #REACTIVE FWD test
+        ping_result = main.FALSE
+        time1 = time.time()
+        ping_result = main.Mininet1.pingall()
+        time2 = time.time()
+        main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+        #uninstall onos-app-fwd
+        main.log.info("Uninstall reactive forwarding app")
+        main.ONOScli1.feature_uninstall("onos-app-fwd")
+        main.ONOScli2.feature_uninstall("onos-app-fwd")
+        main.ONOScli3.feature_uninstall("onos-app-fwd")
+        main.ONOScli4.feature_uninstall("onos-app-fwd")
+        main.ONOScli5.feature_uninstall("onos-app-fwd")
+        main.ONOScli6.feature_uninstall("onos-app-fwd")
+        main.ONOScli7.feature_uninstall("onos-app-fwd")
+
+        main.step("Add  host intents")
+        #TODO:  move the host numbers to params
+        import json
+        intents_json= json.loads(main.ONOScli1.hosts())
+        intent_add_result = main.FALSE
+        for i in range(8,18):
+            main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+            host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+            host2 =  "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+            #NOTE: get host can return None
+            #TODO: handle this
+            host1_id = main.ONOScli1.get_host(host1)['id']
+            host2_id = main.ONOScli1.get_host(host2)['id']
+            tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+            intent_add_result = intent_add_result and tmp_result
+        #TODO Check if intents all exist in datastore
+        #NOTE: Do we need to print this once the test is working?
+        #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+        #    sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+    def CASE4(self,main) :
+        """
+        Ping across added host intents
+        """
+        description = " Ping across added host intents"
+        main.log.report(description)
+        main.case(description)
+        Ping_Result = main.TRUE
+        for i in range(8,18):
+            ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+            Ping_Result = Ping_Result and ping
+            if ping==main.FALSE:
+                main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+            elif ping==main.TRUE:
+                main.log.info("Ping test passed!")
+                Ping_Result = main.TRUE
+        if Ping_Result==main.FALSE:
+            main.log.report("Intents have not been installed correctly, pings failed.")
+        if Ping_Result==main.TRUE:
+            main.log.report("Intents have been installed correctly and verified by pings")
+        utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+                onpass="Intents have been installed correctly and pings work",
+                onfail ="Intents have not been installed correctly, pings failed." )
+
+    def CASE5(self,main) :
+        '''
+        Reading state of ONOS
+        '''
+        import time
+        import json
+        from subprocess import Popen, PIPE
+        from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+        main.log.report("Setting up and gathering data for current state")
+        main.case("Setting up and gathering data for current state")
+        #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+        #We can then compare them with eachother and also with past states
+
+        main.step("Get the Mastership of each switch from each controller")
+        global mastership_state
+        ONOS1_mastership = main.ONOScli1.roles()
+        ONOS2_mastership = main.ONOScli2.roles()
+        ONOS3_mastership = main.ONOScli3.roles()
+        ONOS4_mastership = main.ONOScli4.roles()
+        ONOS5_mastership = main.ONOScli5.roles()
+        ONOS6_mastership = main.ONOScli6.roles()
+        ONOS7_mastership = main.ONOScli7.roles()
+        #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+        if "Error" in ONOS1_mastership or not ONOS1_mastership\
+                or "Error" in ONOS2_mastership or not ONOS2_mastership\
+                or "Error" in ONOS3_mastership or not ONOS3_mastership\
+                or "Error" in ONOS4_mastership or not ONOS4_mastership\
+                or "Error" in ONOS5_mastership or not ONOS5_mastership\
+                or "Error" in ONOS6_mastership or not ONOS6_mastership\
+                or "Error" in ONOS7_mastership or not ONOS7_mastership:
+                    main.log.report("Error in getting ONOS roles")
+                    main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+                    main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+                    main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+                    main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+                    main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+                    main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+                    main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+                    consistent_mastership = main.FALSE
+        elif ONOS1_mastership == ONOS2_mastership\
+                and ONOS1_mastership == ONOS3_mastership\
+                and ONOS1_mastership == ONOS4_mastership\
+                and ONOS1_mastership == ONOS5_mastership\
+                and ONOS1_mastership == ONOS6_mastership\
+                and ONOS1_mastership == ONOS7_mastership:
+                    mastership_state = ONOS1_mastership
+                    consistent_mastership = main.TRUE
+                    main.log.report("Switch roles are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            consistent_mastership = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+                onpass="Switch roles are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of switch roles")
+
+
+        main.step("Get the intents from each controller")
+        global intent_state
+        ONOS1_intents = main.ONOScli1.intents( json_format=True )
+        ONOS2_intents = main.ONOScli2.intents( json_format=True )
+        ONOS3_intents = main.ONOScli3.intents( json_format=True )
+        ONOS4_intents = main.ONOScli4.intents( json_format=True )
+        ONOS5_intents = main.ONOScli5.intents( json_format=True )
+        ONOS6_intents = main.ONOScli6.intents( json_format=True )
+        ONOS7_intents = main.ONOScli7.intents( json_format=True )
+        intent_check = main.FALSE
+        if "Error" in ONOS1_intents or not ONOS1_intents\
+                or "Error" in ONOS2_intents or not ONOS2_intents\
+                or "Error" in ONOS3_intents or not ONOS3_intents\
+                or "Error" in ONOS4_intents or not ONOS4_intents\
+                or "Error" in ONOS5_intents or not ONOS5_intents\
+                or "Error" in ONOS6_intents or not ONOS6_intents\
+                or "Error" in ONOS7_intents or not ONOS7_intents:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+                    main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+                    main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+                    main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+                    main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+                    main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+                    main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+        elif ONOS1_intents == ONOS2_intents\
+                and ONOS1_intents == ONOS3_intents\
+                and ONOS1_intents == ONOS4_intents\
+                and ONOS1_intents == ONOS5_intents\
+                and ONOS1_intents == ONOS6_intents\
+                and ONOS1_intents == ONOS7_intents:
+                    intent_state = ONOS1_intents
+                    intent_check = main.TRUE
+                    main.log.report("Intents are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+                onpass="Intents are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of intents")
+
+
+        main.step("Get the flows from each controller")
+        global flow_state
+        ONOS1_flows = main.ONOScli1.flows( json_format=True )
+        ONOS2_flows = main.ONOScli2.flows( json_format=True )
+        ONOS3_flows = main.ONOScli3.flows( json_format=True )
+        ONOS4_flows = main.ONOScli4.flows( json_format=True )
+        ONOS5_flows = main.ONOScli5.flows( json_format=True )
+        ONOS6_flows = main.ONOScli6.flows( json_format=True )
+        ONOS7_flows = main.ONOScli7.flows( json_format=True )
+        flow_check = main.FALSE
+        if "Error" in ONOS1_flows or not ONOS1_flows\
+                or "Error" in ONOS2_flows or not ONOS2_flows\
+                or "Error" in ONOS3_flows or not ONOS3_flows\
+                or "Error" in ONOS4_flows or not ONOS4_flows\
+                or "Error" in ONOS5_flows or not ONOS5_flows\
+                or "Error" in ONOS6_flows or not ONOS6_flows\
+                or "Error" in ONOS7_flows or not ONOS7_flows:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+                    main.log.warn("ONOS2 flows repsponse: "+ ONOS2_flows)
+                    main.log.warn("ONOS3 flows repsponse: "+ ONOS3_flows)
+                    main.log.warn("ONOS4 flows repsponse: "+ ONOS4_flows)
+                    main.log.warn("ONOS5 flows repsponse: "+ ONOS5_flows)
+                    main.log.warn("ONOS6 flows repsponse: "+ ONOS6_flows)
+                    main.log.warn("ONOS7 flows repsponse: "+ ONOS7_flows)
+        elif len(json.loads(ONOS1_flows)) == len(json.loads(ONOS2_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS3_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS4_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS5_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS6_flows))\
+                and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS7_flows)):
+                #TODO: Do a better check, maybe compare flows on switches?
+                    flow_state = ONOS1_flows
+                    flow_check = main.TRUE
+                    main.log.report("Flow count is consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 flows: "+ json.dumps(json.loads(ONOS1_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 flows: "+ json.dumps(json.loads(ONOS2_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 flows: "+ json.dumps(json.loads(ONOS3_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 flows: "+ json.dumps(json.loads(ONOS4_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 flows: "+ json.dumps(json.loads(ONOS5_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 flows: "+ json.dumps(json.loads(ONOS6_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 flows: "+ json.dumps(json.loads(ONOS7_flows),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=flow_check,
+                onpass="The flow count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different flow counts")
+
+
+        main.step("Get the OF Table entries")
+        global flows
+        flows=[]
+        for i in range(1,29):
+            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+
+        #TODO: Compare switch flow tables with ONOS flow tables
+
+        main.step("Start continuous pings")
+        main.Mininet2.pingLong(src=main.params['PING']['source1'],
+                            target=main.params['PING']['target1'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source2'],
+                            target=main.params['PING']['target2'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source3'],
+                            target=main.params['PING']['target3'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source4'],
+                            target=main.params['PING']['target4'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source5'],
+                            target=main.params['PING']['target5'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source6'],
+                            target=main.params['PING']['target6'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source7'],
+                            target=main.params['PING']['target7'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source8'],
+                            target=main.params['PING']['target8'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source9'],
+                            target=main.params['PING']['target9'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source10'],
+                            target=main.params['PING']['target10'],pingTime=500)
+
+        main.step("Create TestONTopology object")
+        ctrls = []
+        count = 1
+        while True:
+            temp = ()
+            if ('ip' + str(count)) in main.params['CTRL']:
+                temp = temp + (getattr(main,('ONOS' + str(count))),)
+                temp = temp + ("ONOS"+str(count),)
+                temp = temp + (main.params['CTRL']['ip'+str(count)],)
+                temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+                ctrls.append(temp)
+                count = count + 1
+            else:
+                break
+        MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+        main.step("Collecting topology information from ONOS")
+        devices = []
+        devices.append( main.ONOScli1.devices() )
+        devices.append( main.ONOScli2.devices() )
+        devices.append( main.ONOScli3.devices() )
+        devices.append( main.ONOScli4.devices() )
+        devices.append( main.ONOScli5.devices() )
+        devices.append( main.ONOScli6.devices() )
+        devices.append( main.ONOScli7.devices() )
+        '''
+        hosts = []
+        hosts.append( main.ONOScli1.hosts() )
+        hosts.append( main.ONOScli2.hosts() )
+        hosts.append( main.ONOScli3.hosts() )
+        hosts.append( main.ONOScli4.hosts() )
+        hosts.append( main.ONOScli5.hosts() )
+        hosts.append( main.ONOScli6.hosts() )
+        hosts.append( main.ONOScli7.hosts() )
+        '''
+        ports = []
+        ports.append( main.ONOScli1.ports() )
+        ports.append( main.ONOScli2.ports() )
+        ports.append( main.ONOScli3.ports() )
+        ports.append( main.ONOScli4.ports() )
+        ports.append( main.ONOScli5.ports() )
+        ports.append( main.ONOScli6.ports() )
+        ports.append( main.ONOScli7.ports() )
+        links = []
+        links.append( main.ONOScli1.links() )
+        links.append( main.ONOScli2.links() )
+        links.append( main.ONOScli3.links() )
+        links.append( main.ONOScli4.links() )
+        links.append( main.ONOScli5.links() )
+        links.append( main.ONOScli6.links() )
+        links.append( main.ONOScli7.links() )
+
+
+        main.step("Comparing ONOS topology to MN")
+        devices_results = main.TRUE
+        ports_results = main.TRUE
+        links_results = main.TRUE
+        for controller in range(7): #TODO parameterize the number of controllers
+            if devices[controller] or not "Error" in devices[controller]:
+                current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+            else:
+                current_devices_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                    onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+            if ports[controller] or not "Error" in ports[controller]:
+                current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+            else:
+                current_ports_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                    onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+            if links[controller] or not "Error" in links[controller]:
+                current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+            else:
+                current_links_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                    onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+            devices_results = devices_results and current_devices_result
+            ports_results = ports_results and current_ports_result
+            links_results = links_results and current_links_result
+
+        topo_result = devices_results and ports_results and links_results
+        utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+                onpass="Topology Check Test successful",
+                onfail="Topology Check Test NOT successful")
+
+        final_assert = main.TRUE
+        final_assert = final_assert and topo_result and flow_check \
+                and intent_check and consistent_mastership
+        utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+                onpass="State check successful",
+                onfail="State check NOT successful")
+
+
+    def CASE6(self,main) :
+        '''
+        The Failure case. Since this is the Sanity test, we do nothing.
+        '''
+        import time
+        main.log.report("Wait 60 seconds instead of inducing a failure")
+        time.sleep(60)
+        utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+                onpass="Sleeping 60 seconds",
+                onfail="Something is terribly wrong with my math")
+
+    def CASE7(self,main) :
+        '''
+        Check state after ONOS failure
+        '''
+        import os
+        import json
+        main.case("Running ONOS Constant State Tests")
+
+        main.step("Check if switch roles are consistent across all nodes")
+        ONOS1_mastership = main.ONOScli1.roles()
+        ONOS2_mastership = main.ONOScli2.roles()
+        ONOS3_mastership = main.ONOScli3.roles()
+        ONOS4_mastership = main.ONOScli4.roles()
+        ONOS5_mastership = main.ONOScli5.roles()
+        ONOS6_mastership = main.ONOScli6.roles()
+        ONOS7_mastership = main.ONOScli7.roles()
+        #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+        if "Error" in ONOS1_mastership or not ONOS1_mastership\
+                or "Error" in ONOS2_mastership or not ONOS2_mastership\
+                or "Error" in ONOS3_mastership or not ONOS3_mastership\
+                or "Error" in ONOS4_mastership or not ONOS4_mastership\
+                or "Error" in ONOS5_mastership or not ONOS5_mastership\
+                or "Error" in ONOS6_mastership or not ONOS6_mastership\
+                or "Error" in ONOS7_mastership or not ONOS7_mastership:
+                    main.log.error("Error in getting ONOS mastership")
+                    main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+                    main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+                    main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+                    main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+                    main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+                    main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+                    main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+                    consistent_mastership = main.FALSE
+        elif ONOS1_mastership == ONOS2_mastership\
+                and ONOS1_mastership == ONOS3_mastership\
+                and ONOS1_mastership == ONOS4_mastership\
+                and ONOS1_mastership == ONOS5_mastership\
+                and ONOS1_mastership == ONOS6_mastership\
+                and ONOS1_mastership == ONOS7_mastership:
+                    #mastership_state = ONOS1_mastership
+                    consistent_mastership = main.TRUE
+                    main.log.report("Switch roles are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            consistent_mastership = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+                onpass="Switch roles are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of switch roles")
+
+
+        description2 = "Compare switch roles from before failure"
+        main.step(description2)
+
+        current_json = json.loads(ONOS1_mastership)
+        old_json = json.loads(mastership_state)
+        mastership_check = main.TRUE
+        for i in range(1,29):
+            switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+            current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+            old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+            if current == old:
+                mastership_check = mastership_check and main.TRUE
+            else:
+                main.log.warn("Mastership of switch %s changed" % switchDPID)
+                mastership_check = main.FALSE
+        if mastership_check == main.TRUE:
+            main.log.report("Mastership of Switches was not changed")
+        utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+                onpass="Mastership of Switches was not changed",
+                onfail="Mastership of some switches changed")
+        mastership_check = mastership_check and consistent_mastership
+
+
+
+        main.step("Get the intents and compare across all nodes")
+        ONOS1_intents = main.ONOScli1.intents( json_format=True )
+        ONOS2_intents = main.ONOScli2.intents( json_format=True )
+        ONOS3_intents = main.ONOScli3.intents( json_format=True )
+        ONOS4_intents = main.ONOScli4.intents( json_format=True )
+        ONOS5_intents = main.ONOScli5.intents( json_format=True )
+        ONOS6_intents = main.ONOScli6.intents( json_format=True )
+        ONOS7_intents = main.ONOScli7.intents( json_format=True )
+        intent_check = main.FALSE
+        if "Error" in ONOS1_intents or not ONOS1_intents\
+                or "Error" in ONOS2_intents or not ONOS2_intents\
+                or "Error" in ONOS3_intents or not ONOS3_intents\
+                or "Error" in ONOS4_intents or not ONOS4_intents\
+                or "Error" in ONOS5_intents or not ONOS5_intents\
+                or "Error" in ONOS6_intents or not ONOS6_intents\
+                or "Error" in ONOS7_intents or not ONOS7_intents:
+                    main.log.report("Error in getting ONOS intents")
+                    main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+                    main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+                    main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+                    main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+                    main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+                    main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+                    main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+        elif ONOS1_intents == ONOS2_intents\
+                and ONOS1_intents == ONOS3_intents\
+                and ONOS1_intents == ONOS4_intents\
+                and ONOS1_intents == ONOS5_intents\
+                and ONOS1_intents == ONOS6_intents\
+                and ONOS1_intents == ONOS7_intents:
+                    intent_check = main.TRUE
+                    main.log.report("Intents are consistent across all ONOS nodes")
+        else:
+            main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+            main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+                sort_keys=True, indent=4, separators=(',', ': ')))
+        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+                onpass="Intents are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of intents")
+
+        main.step("Compare current intents with intents before the failure")
+        if intent_state == ONOS1_intents:
+            same_intents = main.TRUE
+            main.log.report("Intents are consistent with before failure")
+        #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+        else:
+            same_intents = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+                onpass="Intents are consistent with before failure",
+                onfail="The Intents changed during failure")
+        intent_check = intent_check and same_intents
+
+
+
+        main.step("Get the OF Table entries and compare to before component failure")
+        Flow_Tables = main.TRUE
+        flows2=[]
+        for i in range(28):
+            main.log.info("Checking flow table on s" + str(i+1))
+            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            flows2.append(tmp_flows)
+            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            if Flow_Tables == main.FALSE:
+                main.log.info("Differences in flow table for switch: "+str(i+1))
+                break
+        if Flow_Tables == main.TRUE:
+            main.log.report("No changes were found in the flow tables")
+        utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+                onpass="No changes were found in the flow tables",
+                onfail="Changes were found in the flow tables")
+
+        main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+        #FIXME: This check is always failing. Investigate cause
+        #NOTE:  this may be something to do with file permsissions
+        #       or slight change in format
+        main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
+        Loss_In_Pings = main.FALSE 
+        #NOTE: checkForLoss returns main.FALSE with 0% packet loss
+        for i in range(8,18):
+            main.log.info("Checking for a loss in pings along flow from s" + str(i))
+            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+        if Loss_In_Pings == main.TRUE:
+            main.log.info("Loss in ping detected")
+        elif Loss_In_Pings == main.ERROR:
+            main.log.info("There are multiple mininet process running")
+        elif Loss_In_Pings == main.FALSE:
+            main.log.info("No Loss in the pings")
+            main.log.report("No loss of dataplane connectivity")
+        utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
+                onpass="No Loss of connectivity",
+                onfail="Loss of dataplane connectivity detected")
+
+
+        #TODO:add topology to this or leave as a seperate case?
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = int(result)
+        if result == main.TRUE:
+            main.log.report("Constant State Tests Passed")
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Constant State Tests Passed",
+                onfail="Constant state tests failed")
+
+    def CASE8 (self,main):
+        '''
+        Compare topo
+        '''
+        import sys
+        sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+        from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+        import json
+        import time
+
+        description ="Compare ONOS Topology view to Mininet topology"
+        main.case(description)
+        main.log.report(description)
+        main.step("Create TestONTopology object")
+        ctrls = []
+        count = 1
+        while True:
+            temp = ()
+            if ('ip' + str(count)) in main.params['CTRL']:
+                temp = temp + (getattr(main,('ONOS' + str(count))),)
+                temp = temp + ("ONOS"+str(count),)
+                temp = temp + (main.params['CTRL']['ip'+str(count)],)
+                temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+                ctrls.append(temp)
+                count = count + 1
+            else:
+                break
+        MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+        main.step("Comparing ONOS topology to MN")
+        devices_results = main.TRUE
+        ports_results = main.TRUE
+        links_results = main.TRUE
+        topo_result = main.FALSE
+        start_time = time.time()
+        elapsed = 0
+        count = 0
+        while topo_result == main.FALSE and elapsed < 120:
+            print "cond 1:" + str(topo_result == main.FALSE)
+            print "cond 2:" + str(elapsed < 120)
+            count = count + 1
+            try:
+                main.step("Collecting topology information from ONOS")
+                devices = []
+                devices.append( main.ONOScli1.devices() )
+                devices.append( main.ONOScli2.devices() )
+                devices.append( main.ONOScli3.devices() )
+                devices.append( main.ONOScli4.devices() )
+                devices.append( main.ONOScli5.devices() )
+                devices.append( main.ONOScli6.devices() )
+                devices.append( main.ONOScli7.devices() )
+                '''
+                hosts = []
+                hosts.append( main.ONOScli1.hosts() )
+                hosts.append( main.ONOScli2.hosts() )
+                hosts.append( main.ONOScli3.hosts() )
+                hosts.append( main.ONOScli4.hosts() )
+                hosts.append( main.ONOScli5.hosts() )
+                hosts.append( main.ONOScli6.hosts() )
+                hosts.append( main.ONOScli7.hosts() )
+                '''
+                ports = []
+                ports.append( main.ONOScli1.ports() )
+                ports.append( main.ONOScli2.ports() )
+                ports.append( main.ONOScli3.ports() )
+                ports.append( main.ONOScli4.ports() )
+                ports.append( main.ONOScli5.ports() )
+                ports.append( main.ONOScli6.ports() )
+                ports.append( main.ONOScli7.ports() )
+                links = []
+                links.append( main.ONOScli1.links() )
+                links.append( main.ONOScli2.links() )
+                links.append( main.ONOScli3.links() )
+                links.append( main.ONOScli4.links() )
+                links.append( main.ONOScli5.links() )
+                links.append( main.ONOScli6.links() )
+                links.append( main.ONOScli7.links() )
+
+                for controller in range(7): #TODO parameterize the number of controllers
+                    if devices[controller] or not "Error" in devices[controller]:
+                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                    else:
+                        current_devices_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+                    if ports[controller] or not "Error" in ports[controller]:
+                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                    else:
+                        current_ports_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                    if links[controller] or not "Error" in links[controller]:
+                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                    else:
+                        current_links_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+            except:
+                main.log.error("something went wrong in topo comparison")
+                main.log.warn( repr( devices ) )
+                main.log.warn( repr( ports ) )
+                main.log.warn( repr( links ) )
+
+            devices_results = devices_results and current_devices_result
+            ports_results = ports_results and current_ports_result
+            links_results = links_results and current_links_result
+            topo_result = devices_results and ports_results and links_results
+            elapsed = time.time() - start_time
+        time_threshold = elapsed < 1
+        topo_result = topo_result and time_threshold
+        #TODO make sure this step is non-blocking. IE add a timeout
+        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+                str(elapsed) + " seconds, " + str(count) +" tries" )
+        utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+                onpass="Topology Check Test successful",
+                onfail="Topology Check Test NOT successful")
+        if topo_result == main.TRUE:
+            main.log.report("ONOS topology view matches Mininet topology")
+
+
+    def CASE9 (self,main):
+        '''
+        Link s3-s28 down
+        '''
+        #NOTE: You should probably run a topology check after this
+
+        link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+        description = "Turn off a link to ensure that Link Discovery is working properly"
+        main.log.report(description)
+        main.case(description)
+
+
+        main.step("Kill Link between s3 and s28")
+        Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+        main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+        time.sleep(link_sleep)
+        utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+                onpass="Link down succesful",
+                onfail="Failed to bring link down")
+        #TODO do some sort of check here
+
+    def CASE10 (self,main):
+        '''
+        Link s3-s28 up
+        '''
+        #NOTE: You should probably run a topology check after this
+
+        link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+        description = "Restore a link to ensure that Link Discovery is working properly"
+        main.log.report(description)
+        main.case(description)
+
+        main.step("Bring link between s3 and s28 back up")
+        Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+        main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+        time.sleep(link_sleep)
+        utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+                onpass="Link up succesful",
+                onfail="Failed to bring link up")
+        #TODO do some sort of check here
+
+
+    def CASE11 (self, main) :
+        '''
+        Switch Down
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time
+
+        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+        description = "Killing a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        #TODO: Make this switch parameterizable
+        main.step("Kill s28 ")
+        main.log.report("Deleting s28")
+        #FIXME: use new dynamic topo functions
+        main.Mininet1.del_switch("s28")
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the deleted switch
+        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+        #TODO do some sort of check here
+
+    def CASE12 (self, main) :
+        '''
+        Switch Up
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time
+        #FIXME: use new dynamic topo functions
+        description = "Adding a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        main.step("Add back s28")
+        main.log.report("Adding back s28")
+        main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+        #TODO: New dpid or same? Ask Thomas?
+        main.Mininet1.add_link('s28', 's3')
+        main.Mininet1.add_link('s28', 's6')
+        main.Mininet1.add_link('s28', 'h28')
+        main.Mininet1.assign_sw_controller(sw="28",count=7,
+                ip1=ONOS1_ip,port1=ONOS1_port,
+                ip2=ONOS2_ip,port2=ONOS2_port,
+                ip3=ONOS3_ip,port3=ONOS3_port,
+                ip4=ONOS4_ip,port4=ONOS4_port,
+                ip5=ONOS5_ip,port5=ONOS5_port,
+                ip6=ONOS6_ip,port6=ONOS6_port,
+                ip7=ONOS7_ip,port7=ONOS7_port)
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the added switch
+        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+        #TODO do some sort of check here
+
+    def CASE13 (self, main) :
+        '''
+        Clean up
+        '''
+        import os
+        import time
+        description = "Test Cleanup"
+        main.log.report(description)
+        main.case(description)
+        main.step("Killing tcpdumps")
+        main.Mininet2.stop_tcpdump()
+
+        main.step("Copying MN pcap and ONOS log files to test station")
+        testname = main.TEST
+        #NOTE: MN Pcap file is being saved to ~/packet_captures
+        #       scp this file as MN and TestON aren't necessarily the same vm
+        #FIXME: scp
+        #####mn files
+        #TODO: Load these from params
+        #NOTE: must end in /
+        log_folder = "/opt/onos/log/"
+        log_files = ["karaf.log", "karaf.log.1"]
+        #NOTE: must end in /
+        dst_dir = "~/packet_captures/"
+        for f in log_files:
+            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS1-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS2-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS3-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS4-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS5-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS6-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS7-"+f )
+
+        #std*.log's
+        #NOTE: must end in /
+        log_folder = "/opt/onos/var/"
+        log_files = ["stderr.log", "stdout.log"]
+        #NOTE: must end in /
+        dst_dir = "~/packet_captures/"
+        for f in log_files:
+            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS1-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS2-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS3-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS4-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS5-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS6-"+f )
+            main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS7-"+f )
+
+
+
+
+        #sleep so scp can finish
+        time.sleep(10)
+        main.step("Packing and rotating pcap archives")
+        os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+        #TODO: actually check something here
+        utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+                onpass="Test cleanup successful",
+                onfail="Test cleanup NOT successful")
diff --git a/TestON/tests/HATestSanity/HATestSanity.topo b/TestON/tests/HATestSanity/HATestSanity.topo
new file mode 100644
index 0000000..4d4156c
--- /dev/null
+++ b/TestON/tests/HATestSanity/HATestSanity.topo
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli1>
+
+        <ONOScli2>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli2>
+
+        <ONOScli3>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli3>
+
+
+        <ONOScli4>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli4>
+
+
+        <ONOScli5>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli5>
+
+
+        <ONOScli6>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli6>
+
+
+        <ONOScli7>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>8</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli7>
+
+        <ONOS1>
+            <host>10.128.30.11</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>9</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS1>
+
+        <ONOS2>
+            <host>10.128.30.12</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>10</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS2>
+
+        <ONOS3>
+            <host>10.128.30.13</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>11</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS3>
+
+        <ONOS4>
+            <host>10.128.30.14</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>12</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS4>
+
+        <ONOS5>
+            <host>10.128.30.15</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>13</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS5>
+
+        <ONOS6>
+            <host>10.128.30.16</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>14</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS6>
+
+        <ONOS7>
+            <host>10.128.30.17</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>15</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS7>
+
+        <Mininet1>
+            <host>10.128.30.9</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>MininetCliDriver</type>
+            <connect_order>16</connect_order>
+            <COMPONENTS>
+                #Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+                <arg2> --topo mytopo </arg2>
+                <arg3> </arg3>
+                <controller> remote </controller>
+            </COMPONENTS>
+        </Mininet1>
+
+        <Mininet2>
+            <host>10.128.30.9</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>RemoteMininetDriver</type>
+            <connect_order>17</connect_order>
+            <COMPONENTS>
+                # Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+                <arg2> --topo mytopo --arp</arg2>
+                <controller> remote </controller>
+             </COMPONENTS>
+        </Mininet2>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestSanity/__init__.py b/TestON/tests/HATestSanity/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HATestSanity/__init__.py
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
new file mode 100644
index 0000000..e1e75f9
--- /dev/null
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
@@ -0,0 +1,63 @@
+<PARAMS>
+    <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <ENV>
+    <cellName>HA</cellName>
+    </ENV>
+    <Git>True</Git>
+
+    <CTRL>
+        <ip1>10.128.30.11</ip1>
+        <port1>6633</port1>
+
+        <ip2>10.128.30.12</ip2>
+        <port2>6633</port2>
+
+        <ip3>10.128.30.13</ip3>
+        <port3>6633</port3>
+
+        <ip4>10.128.30.14</ip4>
+        <port4>6633</port4>
+
+        <ip5>10.128.30.15</ip5>
+        <port5>6633</port5>
+
+        <ip6>10.128.30.16</ip6>
+        <port6>6633</port6>
+
+        <ip7>10.128.30.17</ip7>
+        <port7>6633</port7>
+    </CTRL>
+    <TESTONUSER>admin</TESTONUSER>
+    <TESTONIP>10.128.30.10</TESTONIP>
+    <PING>
+        <source1>h8</source1>
+        <source2>h9</source2>
+        <source3>h10</source3>
+        <source4>h11</source4>
+        <source5>h12</source5>
+        <source6>h13</source6>
+        <source7>h14</source7>
+        <source8>h15</source8>
+        <source9>h16</source9>
+        <source10>h17</source10>
+        <target1>10.0.0.18</target1>
+        <target2>10.0.0.19</target2>
+        <target3>10.0.0.20</target3>
+        <target4>10.0.0.21</target4>
+        <target5>10.0.0.22</target5>
+        <target6>10.0.0.23</target6>
+        <target7>10.0.0.24</target7>
+        <target8>10.0.0.25</target8>
+        <target9>10.0.0.26</target9>
+        <target10>10.0.0.27</target10>
+    </PING>
+    <timers>
+        <LinkDiscovery>2</LinkDiscovery>
+        <SwitchDiscovery>2</SwitchDiscovery>
+    </timers>
+    <MNtcpdump>
+        <intf>eth0</intf>
+        <port> </port>
+        <folder>~/packet_captures/</folder>
+    </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
new file mode 100644
index 0000000..e6f2fde
--- /dev/null
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -0,0 +1,814 @@
+'''
+Description: This test is to determine if a single
+    instance ONOS 'cluster' can handle a restart
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case. Since this is the Sanity test, we do nothing.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+'''
+class SingleInstanceHATestRestart:
+
+    def __init__(self) :
+        self.default = ''
+
+    def CASE1(self,main) :
+        '''
+        CASE1 is to compile ONOS and push it to the test machines
+
+        Startup sequence:
+        git pull
+        mvn clean install
+        onos-package
+        cell <name>
+        onos-verify-cell
+        NOTE: temporary - onos-remove-raft-logs
+        onos-install -f
+        onos-wait-for-start
+        '''
+        import time
+        main.log.report("ONOS Single node cluster restart HA test - initialization")
+        main.case("Setting up test environment")
+
+        # load some vairables from the params file
+        PULL_CODE = False
+        if main.params['Git'] == 'True':
+            PULL_CODE = True
+        cell_name = main.params['ENV']['cellName']
+
+        #set global variables
+        global ONOS1_ip
+        global ONOS1_port
+        global ONOS2_ip
+        global ONOS2_port
+        global ONOS3_ip
+        global ONOS3_port
+        global ONOS4_ip
+        global ONOS4_port
+        global ONOS5_ip
+        global ONOS5_port
+        global ONOS6_ip
+        global ONOS6_port
+        global ONOS7_ip
+        global ONOS7_port
+
+        ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS1_port = main.params['CTRL']['port1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS2_port = main.params['CTRL']['port2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS3_port = main.params['CTRL']['port3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS4_port = main.params['CTRL']['port4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS5_port = main.params['CTRL']['port5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS6_port = main.params['CTRL']['port6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+        ONOS7_port = main.params['CTRL']['port7']
+
+
+        main.step("Applying cell variable to environment")
+        cell_result = main.ONOSbench.set_cell(cell_name)
+        verify_result = main.ONOSbench.verify_cell()
+
+        #FIXME:this is short term fix
+        main.log.report("Removing raft logs")
+        main.ONOSbench.onos_remove_raft_logs()
+        main.log.report("Uninstalling ONOS")
+        main.ONOSbench.onos_uninstall(ONOS1_ip)
+        main.ONOSbench.onos_uninstall(ONOS2_ip)
+        main.ONOSbench.onos_uninstall(ONOS3_ip)
+        main.ONOSbench.onos_uninstall(ONOS4_ip)
+        main.ONOSbench.onos_uninstall(ONOS5_ip)
+        main.ONOSbench.onos_uninstall(ONOS6_ip)
+        main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+        clean_install_result = main.TRUE
+        git_pull_result = main.TRUE
+
+        main.step("Compiling the latest version of ONOS")
+        if PULL_CODE:
+            main.step("Git checkout and pull master")
+            main.ONOSbench.git_checkout("master")
+            git_pull_result = main.ONOSbench.git_pull()
+
+            main.step("Using mvn clean & install")
+            clean_install_result = main.TRUE
+            if git_pull_result == main.TRUE:
+                clean_install_result = main.ONOSbench.clean_install()
+            else:
+                main.log.warn("Did not pull new code so skipping mvn "+ \
+                        "clean install")
+        main.ONOSbench.get_version(report=True)
+
+        main.step("Creating ONOS package")
+        package_result = main.ONOSbench.onos_package()
+
+        main.step("Installing ONOS package")
+        onos1_install_result = main.ONOSbench.onos_install(options="-f",
+                node=ONOS1_ip)
+
+
+        main.step("Checking if ONOS is up yet")
+        #TODO: Refactor
+        # check bundle:list?
+        #this should be enough for ONOS to start
+        time.sleep(60)
+        onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+        if not onos1_isup:
+            main.log.report("ONOS1 didn't start!")
+        # TODO: if it becomes an issue, we can retry this step  a few times
+
+
+        cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+
+        main.step("Start Packet Capture MN")
+        main.Mininet2.start_tcpdump(
+                str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+                intf = main.params['MNtcpdump']['intf'],
+                port = main.params['MNtcpdump']['port'])
+
+
+        case1_result = (clean_install_result and package_result and
+                cell_result and verify_result and onos1_install_result and
+                onos1_isup and cli1_results)
+
+        utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+                onpass="Test startup successful",
+                onfail="Test startup NOT successful")
+
+
+        if case1_result==main.FALSE:
+            main.cleanup()
+            main.exit()
+
+    def CASE2(self,main) :
+        '''
+        Assign mastership to controllers
+        '''
+        import time
+        import json
+        import re
+
+        main.log.report("Assigning switches to controllers")
+        main.case("Assigning Controllers")
+        main.step("Assign switches to controllers")
+
+        for i in range (1,29):
+           main.Mininet1.assign_sw_controller(sw=str(i),
+                    ip1=ONOS1_ip,port1=ONOS1_port)
+
+        mastership_check = main.TRUE
+        for i in range (1,29):
+            response = main.Mininet1.get_sw_controller("s"+str(i))
+            try:
+                main.log.info(str(response))
+            except:
+                main.log.info(repr(response))
+            if re.search("tcp:"+ONOS1_ip,response):
+                mastership_check = mastership_check and main.TRUE
+            else:
+                mastership_check = main.FALSE
+        if mastership_check == main.TRUE:
+            main.log.report("Switch mastership assigned correctly")
+        utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+                onpass="Switch mastership assigned correctly",
+                onfail="Switches not assigned correctly to controllers")
+
+        #TODO: If assign roles is working reliably then manually 
+        #   assign mastership to the controller we want
+
+
+    def CASE3(self,main) :
+        """
+        Assign intents
+
+        """
+        import time
+        import json
+        import re
+        main.log.report("Adding host intents")
+        main.case("Adding host Intents")
+
+        main.step("Discovering  Hosts( Via pingall for now)")
+        #FIXME: Once we have a host discovery mechanism, use that instead
+
+        #REACTIVE FWD test
+        ping_result = main.FALSE
+        time1 = time.time()
+        ping_result = main.Mininet1.pingall()
+        time2 = time.time()
+        main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+        #uninstall onos-app-fwd
+        main.log.info("Uninstall reactive forwarding app")
+        main.ONOScli1.feature_uninstall("onos-app-fwd")
+
+        main.step("Add  host intents")
+        #TODO:  move the host numbers to params
+        import json
+        intents_json= json.loads(main.ONOScli1.hosts())
+        intent_add_result = main.FALSE
+        for i in range(8,18):
+            main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+            host1 =  "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+            host2 =  "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+            #NOTE: get host can return None
+            #TODO: handle this
+            host1_id = main.ONOScli1.get_host(host1)['id']
+            host2_id = main.ONOScli1.get_host(host2)['id']
+            tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+            intent_add_result = intent_add_result and tmp_result
+        #TODO Check if intents all exist in datastore
+        #NOTE: Do we need to print this once the test is working?
+        #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+        #    sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+    def CASE4(self,main) :
+        """
+        Ping across added host intents
+        """
+        description = " Ping across added host intents"
+        main.log.report(description)
+        main.case(description)
+        Ping_Result = main.TRUE
+        for i in range(8,18):
+            ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+            Ping_Result = Ping_Result and ping
+            if ping==main.FALSE:
+                main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+            elif ping==main.TRUE:
+                main.log.info("Ping test passed!")
+                Ping_Result = main.TRUE
+        if Ping_Result==main.FALSE:
+            main.log.report("Intents have not been installed correctly, pings failed.")
+        if Ping_Result==main.TRUE:
+            main.log.report("Intents have been installed correctly and verified by pings")
+        utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+                onpass="Intents have been installed correctly and pings work",
+                onfail ="Intents have not been installed correctly, pings failed." )
+
+    def CASE5(self,main) :
+        '''
+        Reading state of ONOS
+        '''
+        import time
+        import json
+        from subprocess import Popen, PIPE
+        from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+        main.log.report("Setting up and gathering data for current state")
+        main.case("Setting up and gathering data for current state")
+        #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+        #We can then compare them with eachother and also with past states
+
+        main.step("Get the Mastership of each switch from each controller")
+        global mastership_state
+        ONOS1_mastership = main.ONOScli1.roles()
+        #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+        #TODO: Make this a meaningful check
+        if "Error" in ONOS1_mastership or not ONOS1_mastership:
+            main.log.report("Error in getting ONOS roles")
+            main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+            consistent_mastership = main.FALSE
+        else:
+            mastership_state = ONOS1_mastership
+            consistent_mastership = main.TRUE
+            main.log.report("Switch roles are consistent across all ONOS nodes")
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+                onpass="Switch roles are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of switch roles")
+
+
+        main.step("Get the intents from each controller")
+        global intent_state
+        ONOS1_intents = main.ONOScli1.intents( json_format=True )
+        intent_check = main.FALSE
+        if "Error" in ONOS1_intents or not ONOS1_intents:
+            main.log.report("Error in getting ONOS intents")
+            main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+        else:
+            intent_check = main.TRUE
+            main.log.report("Intents are consistent across all ONOS nodes")
+        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+                onpass="Intents are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of intents")
+
+
+        main.step("Get the flows from each controller")
+        global flow_state
+        ONOS1_flows = main.ONOScli1.flows( json_format=True )
+        flow_check = main.FALSE
+        if "Error" in ONOS1_flows or not ONOS1_flows:
+            main.log.report("Error in getting ONOS intents")
+            main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+        else:
+            #TODO: Do a better check, maybe compare flows on switches?
+            flow_state = ONOS1_flows
+            flow_check = main.TRUE
+            main.log.report("Flow count is consistent across all ONOS nodes")
+        utilities.assert_equals(expect = main.TRUE,actual=flow_check,
+                onpass="The flow count is consistent across all ONOS nodes",
+                onfail="ONOS nodes have different flow counts")
+
+
+        main.step("Get the OF Table entries")
+        global flows
+        flows=[]
+        for i in range(1,29):
+            flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+
+        #TODO: Compare switch flow tables with ONOS flow tables
+
+        main.step("Start continuous pings")
+        main.Mininet2.pingLong(src=main.params['PING']['source1'],
+                            target=main.params['PING']['target1'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source2'],
+                            target=main.params['PING']['target2'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source3'],
+                            target=main.params['PING']['target3'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source4'],
+                            target=main.params['PING']['target4'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source5'],
+                            target=main.params['PING']['target5'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source6'],
+                            target=main.params['PING']['target6'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source7'],
+                            target=main.params['PING']['target7'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source8'],
+                            target=main.params['PING']['target8'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source9'],
+                            target=main.params['PING']['target9'],pingTime=500)
+        main.Mininet2.pingLong(src=main.params['PING']['source10'],
+                            target=main.params['PING']['target10'],pingTime=500)
+
+        main.step("Create TestONTopology object")
+        ctrls = []
+        count = 1
+        temp = ()
+        temp = temp + (getattr(main,('ONOS' + str(count))),)
+        temp = temp + ("ONOS"+str(count),)
+        temp = temp + (main.params['CTRL']['ip'+str(count)],)
+        temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+        ctrls.append(temp)
+        MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+        main.step("Collecting topology information from ONOS")
+        devices = []
+        devices.append( main.ONOScli1.devices() )
+        '''
+        hosts = []
+        hosts.append( main.ONOScli1.hosts() )
+        '''
+        ports = []
+        ports.append( main.ONOScli1.ports() )
+        links = []
+        links.append( main.ONOScli1.links() )
+
+
+        main.step("Comparing ONOS topology to MN")
+        devices_results = main.TRUE
+        ports_results = main.TRUE
+        links_results = main.TRUE
+        for controller in range(1): #TODO parameterize the number of controllers
+            if devices[controller] or not "Error" in devices[controller]:
+                current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+            else:
+                current_devices_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                    onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+            if ports[controller] or not "Error" in ports[controller]:
+                current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+            else:
+                current_ports_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                    onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+            if links[controller] or not "Error" in links[controller]:
+                current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+            else:
+                current_links_result = main.FALSE
+            utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                    onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                    onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+            devices_results = devices_results and current_devices_result
+            ports_results = ports_results and current_ports_result
+            links_results = links_results and current_links_result
+
+        topo_result = devices_results and ports_results and links_results
+        utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+                onpass="Topology Check Test successful",
+                onfail="Topology Check Test NOT successful")
+
+        final_assert = main.TRUE
+        final_assert = final_assert and topo_result and flow_check \
+                and intent_check and consistent_mastership
+        utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+                onpass="State check successful",
+                onfail="State check NOT successful")
+
+
+    def CASE6(self,main) :
+        '''
+        The Failure case.
+        '''
+        import time
+
+        main.log.report("Restart ONOS node")
+        main.log.case("Restart ONOS node")
+        main.ONOSbench.onos_kill(ONOS1_ip)
+        start = time.time()
+
+        main.step("Checking if ONOS is up yet")
+        count = 0
+        while count < 10
+            onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+            if onos1_isup == main.TRUE:
+                elapsed = time.time() - start
+                break
+            else:
+                count = count + 1
+
+        cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
+
+        case_results = main.TRUE and onosi1_isup and cli_result
+        utilities.assert_equals(expect=main.TRUE, actual=case_results,
+                onpass="ONOS restart successful",
+                onfail="ONOS restart NOT successful")
+        main.log.info("ONOS took %s seconds to restart" % str(elapsed) )
+
+    def CASE7(self,main) :
+        '''
+        Check state after ONOS failure
+        '''
+        import os
+        import json
+        main.case("Running ONOS Constant State Tests")
+
+        main.step("Check if switch roles are consistent across all nodes")
+        ONOS1_mastership = main.ONOScli1.roles()
+        #FIXME: Refactor this whole case for single instance
+        #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+        if "Error" in ONOS1_mastership or not ONOS1_mastership:
+            main.log.report("Error in getting ONOS mastership")
+            main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+            consistent_mastership = main.FALSE
+        else:
+            consistent_mastership = main.TRUE
+            main.log.report("Switch roles are consistent across all ONOS nodes")
+        utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+                onpass="Switch roles are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of switch roles")
+
+
+        description2 = "Compare switch roles from before failure"
+        main.step(description2)
+
+        current_json = json.loads(ONOS1_mastership)
+        old_json = json.loads(mastership_state)
+        mastership_check = main.TRUE
+        for i in range(1,29):
+            switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+            current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+            old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+            if current == old:
+                mastership_check = mastership_check and main.TRUE
+            else:
+                main.log.warn("Mastership of switch %s changed" % switchDPID)
+                mastership_check = main.FALSE
+        if mastership_check == main.TRUE:
+            main.log.report("Mastership of Switches was not changed")
+        utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+                onpass="Mastership of Switches was not changed",
+                onfail="Mastership of some switches changed")
+        mastership_check = mastership_check and consistent_mastership
+
+
+
+        main.step("Get the intents and compare across all nodes")
+        ONOS1_intents = main.ONOScli1.intents( json_format=True )
+        intent_check = main.FALSE
+        if "Error" in ONOS1_intents or not ONOS1_intents:
+            main.log.report("Error in getting ONOS intents")
+            main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+        else:
+            intent_state = ONOS1_intents
+            intent_check = main.TRUE
+            main.log.report("Intents are consistent across all ONOS nodes")
+        utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+                onpass="Intents are consistent across all ONOS nodes",
+                onfail="ONOS nodes have different views of intents")
+
+        main.step("Compare current intents with intents before the failure")
+        if intent_state == ONOS1_intents:
+            same_intents = main.TRUE
+            main.log.report("Intents are consistent with before failure")
+        #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+        else:
+            same_intents = main.FALSE
+        utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+                onpass="Intents are consistent with before failure",
+                onfail="The Intents changed during failure")
+        intent_check = intent_check and same_intents
+
+
+
+        main.step("Get the OF Table entries and compare to before component failure")
+        Flow_Tables = main.TRUE
+        flows2=[]
+        for i in range(28):
+            main.log.info("Checking flow table on s" + str(i+1))
+            tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+            flows2.append(tmp_flows)
+            Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+            if Flow_Tables == main.FALSE:
+                main.log.info("Differences in flow table for switch: "+str(i+1))
+                break
+        if Flow_Tables == main.TRUE:
+            main.log.report("No changes were found in the flow tables")
+        utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+                onpass="No changes were found in the flow tables",
+                onfail="Changes were found in the flow tables")
+
+        main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+        #FIXME: This check is always failing. Investigate cause
+        #NOTE:  this may be something to do with file permsissions
+        #       or slight change in format
+        main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
+        Loss_In_Pings = main.FALSE 
+        #NOTE: checkForLoss returns main.FALSE with 0% packet loss
+        for i in range(8,18):
+            main.log.info("Checking for a loss in pings along flow from s" + str(i))
+            Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+        if Loss_In_Pings == main.TRUE:
+            main.log.info("Loss in ping detected")
+        elif Loss_In_Pings == main.ERROR:
+            main.log.info("There are multiple mininet process running")
+        elif Loss_In_Pings == main.FALSE:
+            main.log.info("No Loss in the pings")
+            main.log.report("No loss of dataplane connectivity")
+        utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
+                onpass="No Loss of connectivity",
+                onfail="Loss of dataplane connectivity detected")
+
+
+        #TODO:add topology to this or leave as a seperate case?
+        result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+        result = int(result)
+        if result == main.TRUE:
+            main.log.report("Constant State Tests Passed")
+        utilities.assert_equals(expect=main.TRUE,actual=result,
+                onpass="Constant State Tests Passed",
+                onfail="Constant state tests failed")
+
+    def CASE8 (self,main):
+        '''
+        Compare topo
+        '''
+        import sys
+        sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+        from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+        import json
+        import time
+
+        description ="Compare ONOS Topology view to Mininet topology"
+        main.case(description)
+        main.log.report(description)
+        main.step("Create TestONTopology object")
+        ctrls = []
+        count = 1
+        temp = ()
+        temp = temp + (getattr(main,('ONOS' + str(count))),)
+        temp = temp + ("ONOS"+str(count),)
+        temp = temp + (main.params['CTRL']['ip'+str(count)],)
+        temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+        ctrls.append(temp)
+        MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+        main.step("Comparing ONOS topology to MN")
+        devices_results = main.TRUE
+        ports_results = main.TRUE
+        links_results = main.TRUE
+        topo_result = main.FALSE
+        start_time = time.time()
+        elapsed = 0
+        count = 0
+        while topo_result == main.FALSE and elapsed < 120:
+            count = count + 1
+            try:
+                main.step("Collecting topology information from ONOS")
+                devices = []
+                devices.append( main.ONOScli1.devices() )
+                '''
+                hosts = []
+                hosts.append( main.ONOScli1.hosts() )
+                '''
+                ports = []
+                ports.append( main.ONOScli1.ports() )
+                links = []
+                links.append( main.ONOScli1.links() )
+                for controller in range(1): #TODO parameterize the number of controllers
+                    if devices[controller] or not "Error" in devices[controller]:
+                        current_devices_result =  main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+                    else:
+                        current_devices_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+                            onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+                    if ports[controller] or not "Error" in ports[controller]:
+                        current_ports_result =  main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+                    else:
+                        current_ports_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+                            onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+                    if links[controller] or not "Error" in links[controller]:
+                        current_links_result =  main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+                    else:
+                        current_links_result = main.FALSE
+                    utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+                            onpass="ONOS"+str(int(controller+1))+" links view is correct",
+                            onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+            except:
+                main.log.error("something went wrong in topo comparison")
+                main.log.warn( repr( devices ) )
+                main.log.warn( repr( ports ) )
+                main.log.warn( repr( links ) )
+
+            devices_results = devices_results and current_devices_result
+            ports_results = ports_results and current_ports_result
+            links_results = links_results and current_links_result
+            elapsed = time.time() - start_time
+        time_threshold = elapsed < 1
+        topo_result = devices_results and ports_results and links_results and time_threshold
+        #TODO make sure this step is non-blocking. IE add a timeout
+        main.log.report("Very crass estimate for topology discovery/convergence: " +\
+                str(elapsed) + " seconds, " + str(count) +" tries" )
+        utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+                onpass="Topology Check Test successful",
+                onfail="Topology Check Test NOT successful")
+        if topo_result == main.TRUE:
+            main.log.report("ONOS topology view matches Mininet topology")
+
+
+    def CASE9 (self,main):
+        '''
+        Link s3-s28 down
+        '''
+        #NOTE: You should probably run a topology check after this
+
+        link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+        description = "Turn off a link to ensure that Link Discovery is working properly"
+        main.log.report(description)
+        main.case(description)
+
+
+        main.step("Kill Link between s3 and s28")
+        Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+        main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+        time.sleep(link_sleep)
+        utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+                onpass="Link down succesful",
+                onfail="Failed to bring link down")
+        #TODO do some sort of check here
+
+    def CASE10 (self,main):
+        '''
+        Link s3-s28 up
+        '''
+        #NOTE: You should probably run a topology check after this
+
+        link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+        description = "Restore a link to ensure that Link Discovery is working properly"
+        main.log.report(description)
+        main.case(description)
+
+        main.step("Bring link between s3 and s28 back up")
+        Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+        main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+        time.sleep(link_sleep)
+        utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+                onpass="Link up succesful",
+                onfail="Failed to bring link up")
+        #TODO do some sort of check here
+
+
+    def CASE11 (self, main) :
+        '''
+        Switch Down
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time
+
+        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+        description = "Killing a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        #TODO: Make this switch parameterizable
+        main.step("Kill s28 ")
+        main.log.report("Deleting s28")
+        #FIXME: use new dynamic topo functions
+        main.Mininet1.del_switch("s28")
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the deleted switch
+        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+        #TODO do some sort of check here
+
+    def CASE12 (self, main) :
+        '''
+        Switch Up
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time
+        #FIXME: use new dynamic topo functions
+        description = "Adding a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        main.step("Add back s28")
+        main.log.report("Adding back s28")
+        main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+        #TODO: New dpid or same? Ask Thomas?
+        main.Mininet1.add_link('s28', 's3')
+        main.Mininet1.add_link('s28', 's6')
+        main.Mininet1.add_link('s28', 'h28')
+        main.Mininet1.assign_sw_controller(sw="28",
+                ip1=ONOS1_ip,port1=ONOS1_port)
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the added switch
+        main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+        #TODO do some sort of check here
+
+    def CASE13 (self, main) :
+        '''
+        Clean up
+        '''
+        import os
+        import time
+        description = "Test Cleanup"
+        main.log.report(description)
+        main.case(description)
+        main.step("Killing tcpdumps")
+        main.Mininet2.stop_tcpdump()
+
+        main.step("Copying MN pcap and ONOS log files to test station")
+        testname = main.TEST
+        #NOTE: MN Pcap file is being saved to ~/packet_captures
+        #       scp this file as MN and TestON aren't necessarily the same vm
+        #FIXME: scp
+        #####mn files
+        #TODO: Load these from params
+        #NOTE: must end in /
+        log_folder = "/opt/onos/log/"
+        log_files = ["karaf.log", "karaf.log.1"]
+        #NOTE: must end in /
+        dst_dir = "~/packet_captures/"
+        for f in log_files:
+            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS1-"+f )
+
+        #std*.log's
+        #NOTE: must end in /
+        log_folder = "/opt/onos/var/"
+        log_files = ["stderr.log", "stdout.log"]
+        #NOTE: must end in /
+        dst_dir = "~/packet_captures/"
+        for f in log_files:
+            main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+                    dst_dir + str(testname) + "-ONOS1-"+f )
+
+
+        #sleep so scp can finish
+        time.sleep(10)
+        main.step("Packing and rotating pcap archives")
+        os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+        #TODO: actually check something here
+        utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+                onpass="Test cleanup successful",
+                onfail="Test cleanup NOT successful")
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.topo b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.topo
new file mode 100644
index 0000000..4d4156c
--- /dev/null
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.topo
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli1>
+
+        <ONOScli2>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli2>
+
+        <ONOScli3>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli3>
+
+
+        <ONOScli4>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli4>
+
+
+        <ONOScli5>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli5>
+
+
+        <ONOScli6>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli6>
+
+
+        <ONOScli7>
+            <host>10.128.30.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>8</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli7>
+
+        <ONOS1>
+            <host>10.128.30.11</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>9</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS1>
+
+        <ONOS2>
+            <host>10.128.30.12</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>10</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS2>
+
+        <ONOS3>
+            <host>10.128.30.13</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>11</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS3>
+
+        <ONOS4>
+            <host>10.128.30.14</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>12</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS4>
+
+        <ONOS5>
+            <host>10.128.30.15</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>13</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS5>
+
+        <ONOS6>
+            <host>10.128.30.16</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>14</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS6>
+
+        <ONOS7>
+            <host>10.128.30.17</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosDriver</type>
+            <connect_order>15</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS7>
+
+        <Mininet1>
+            <host>10.128.30.9</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>MininetCliDriver</type>
+            <connect_order>16</connect_order>
+            <COMPONENTS>
+                #Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+                <arg2> --topo mytopo </arg2>
+                <arg3> </arg3>
+                <controller> remote </controller>
+            </COMPONENTS>
+        </Mininet1>
+
+        <Mininet2>
+            <host>10.128.30.9</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>RemoteMininetDriver</type>
+            <connect_order>17</connect_order>
+            <COMPONENTS>
+                # Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+                <arg2> --topo mytopo --arp</arg2>
+                <controller> remote </controller>
+             </COMPONENTS>
+        </Mininet2>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/SingleInstanceHATestRestart/__init__.py b/TestON/tests/SingleInstanceHATestRestart/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/SingleInstanceHATestRestart/__init__.py