diff --git a/TestON/core/Thread.py b/TestON/core/Thread.py
index da30339..fcc06b5 100644
--- a/TestON/core/Thread.py
+++ b/TestON/core/Thread.py
@@ -18,6 +18,8 @@
             if self.target is not None:
                 self.result = self.target( *self.args, **self.kwargs )
         except Exception as e:
-            print "Thread-" + str( self.threadID ) + " '" + self.name + "'"\
-                  ":something went wrong with " + self.target + " method"
+            print "ThreadID:" + str( self.threadID ) + ", Name:" +\
+                  self.name + "- something went wrong with " +\
+                  str( self.target.im_class ) + "." +\
+                  str( self.target.im_func ) + " method: "
             print e
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 667f99c..271c0dd 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -32,6 +32,7 @@
 import new
 import xmldict
 import importlib
+import os
 module = new.module("test")
 import openspeak
 global path, drivers_path, core_path, tests_path,logs_path
@@ -78,8 +79,6 @@
         self.testResult = "Summary"
         self.stepName = ""
         self.stepCache = ""
-        # make this into two lists? one for step names, one for results?
-        # this way, the case result could be a true AND of these results
         self.EXPERIMENTAL_MODE = False
         self.test_target = None
         self.lastcommand = None
@@ -164,11 +163,19 @@
         driverClass = getattr(driverModule, driverName)
         driverObject = driverClass()
 
+        try:
+            self.componentDictionary[component]['host'] = os.environ[str( self.componentDictionary[component]['host'])]
+        except KeyError:
+            self.log.info("Missing OC environment variable! Using stored IPs")
+        except Exception as inst:
+            self.log.error("Uncaught exception: " + str(inst))
+
         connect_result = driverObject.connect(user_name = self.componentDictionary[component]['user'] if ('user' in self.componentDictionary[component].keys()) else getpass.getuser(),
                                               ip_address= self.componentDictionary[component]['host'] if ('host' in self.componentDictionary[component].keys()) else 'localhost',
                                               pwd = self.componentDictionary[component]['password'] if ('password' in self.componentDictionary[component].keys()) else 'changeme',
                                               port = self.componentDictionary[component]['port'] if ('port' in self.componentDictionary[component].keys()) else None,
                                               options = driver_options)
+
         if not connect_result:
             self.log.error("Exiting form the test execution because the connecting to the "+component+" component failed.")
             self.exit()
@@ -251,13 +258,14 @@
             self.log.summary( self.caseExplaination )
             self.log.wiki( "<ul>" )
             for line in self.stepCache.splitlines():
-                self.log.error( repr(line) )
                 if re.search( " - PASS$", line ):
                     self.log.wiki( "<li>" + line + "  <ac:emoticon ac:name=\"tick\" /></li>\n" )
                 elif re.search( " - FAIL$", line ):
                     self.log.wiki( "<li>" + line + "  <ac:emoticon ac:name=\"cross\" /></li>\n" )
                 elif re.search( " - No Result$", line ):
                     self.log.wiki( "<li>" + line + "  <ac:emoticon ac:name=\"warning\" /></li>\n" )
+                else:  # Should only be on fail message
+                    self.log.wiki( "<ul><li>" + line + "</li></ul>\n" )
             self.log.wiki( "</ul>" )
             self.log.summary( self.stepCache )
             self.stepCache = ""
@@ -268,38 +276,38 @@
             try :
                 step = stepList[self.stepCount]
                 self.STEPRESULT = self.NORESULT
+                self.onFailMsg = "\t\tNo on fail message given"
                 exec code[testCaseNumber][step] in module.__dict__
                 self.stepCount = self.stepCount + 1
                 if step > 0:
                     self.stepCache += "\t"+str(testCaseNumber)+"."+str(step)+" "+self.stepName+" - "
                     if self.STEPRESULT == self.TRUE:
                         self.stepCache += "PASS\n"
-                        #self.stepCache += "PASS  <ac:emoticon ac:name=\"tick\" /></li>\n"
                     elif self.STEPRESULT == self.FALSE:
                         self.stepCache += "FAIL\n"
-                        #self.stepCache += "FAIL  <ac:emoticon ac:name=\"cross\" /></li>\n"
+                        # TODO: Print the on-fail statement here
+                        self.stepCache += "\t\t" + self.onFailMsg + "\n"
                     else:
                         self.stepCache += "No Result\n"
-                        #self.stepCache += "No Result  <ac:emoticon ac:name=\"warning\" /></li>\n"
                     self.stepResults.append(self.STEPRESULT)
-            except TypeError,e:
-                print "\nException in the following section of code: " +\
-                      str(testCaseNumber) + "." + str(step) + ": " +\
-                      self.stepName
+            except StandardError as e:
+                self.log.exception( "\nException in the following section of" +
+                                    " code: " + str(testCaseNumber) + "." +
+                                    str(step) + ": " + self.stepName )
                 #print code[testCaseNumber][step]
                 self.stepCount = self.stepCount + 1
-                self.log.exception(e)
                 self.logger.updateCaseResults(self)
                 #WIKI results
                 self.log.wiki( "<ul>" )
-                for line in self.stepCachesplitlines():
-                    self.log.error( repr(line) )
+                for line in self.stepCache.splitlines():
                     if re.search( " - PASS$", line ):
                         self.log.wiki( "<li>" + line + "  <ac:emoticon ac:name=\"tick\" /></li>\n" )
                     elif re.search( " - FAIL$", line ):
                         self.log.wiki( "<li>" + line + "  <ac:emoticon ac:name=\"cross\" /></li>\n" )
                     elif re.search( " - No Result$", line ):
                         self.log.wiki( "<li>" + line + "  <ac:emoticon ac:name=\"warning\" /></li>\n" )
+                    else:  # Should only be on fail message
+                        self.log.wiki( "<ul><li>" + line + "</li></ul>\n" )
                 self.log.wiki( "</ul>" )
                 #summary results
                 self.log.summary( self.stepCache )
diff --git a/TestON/core/testparser.py b/TestON/core/testparser.py
index 37f50f0..24b1ca2 100644
--- a/TestON/core/testparser.py
+++ b/TestON/core/testparser.py
@@ -16,7 +16,7 @@
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with TestON.  If not, see <http://www.gnu.org/licenses/>.		
+    along with TestON.  If not, see <http://www.gnu.org/licenses/>.
 
 
 '''
@@ -26,21 +26,21 @@
     def __init__(self,testFile):
         try :
             testFileHandler = open(testFile, 'r')
-        except IOError: 
+        except IOError:
             print "No such file "+testFile
             sys.exit(0)
- 
+
         testFileList = testFileHandler.readlines()
-        self.testscript = testFileList              
+        self.testscript = testFileList
         self.caseCode = {}
         self.caseBlock = ''
         self.statementsList = []
-        index = 0 
+        index = 0
         self.statementsList = []
         #initialSpaces = len(line) -len(line.lstrip())
         while index < len(testFileList):
             testFileList[index] = re.sub("^\s{8}|^\s{4}", "", testFileList[index])
-            # Skip multiline comments 
+            # Skip multiline comments
             if re.match('^(\'\'\')|^(\"\"\")',testFileList[index],0) :
                 index = index + 1
                 try :
@@ -48,16 +48,15 @@
                         index = index + 1
                 except IndexError,e:
                     print ''
-                    
 
-            # skip empty lines and single line comments 
+            # skip empty lines and single line comments
             elif not re.match('#|^\s*$',testFileList[index],0):
                 self.statementsList.append(testFileList[index])
             index = index + 1
-    
+
     def case_code(self):
-        index = 0 
-        statementsList = self.statementsList       
+        index = 0
+        statementsList = self.statementsList
         while index < len(statementsList):
             #print statementsList[index]
             m= re.match('def\s+CASE(\d+)',statementsList[index],0)
@@ -76,18 +75,16 @@
                 except IndexError,e:
                     #print 'IndexError'
                     print ''
-    
                 self.caseCode [str(m.group(1))] = self.caseBlock
                 #print "Case CODE "+self.caseCode [str(m.group(1))]
             index = index + 1
-        
-        return self.caseCode 
-    
+        return self.caseCode
+
     def step_code(self,caseStatements):
         index = 0
-        step = 0 
-        stepCode = {}  
-        step_flag = False    
+        step = 0
+        stepCode = {}
+        step_flag = False
         while index < len(caseStatements):
             m= re.match('main\.step',caseStatements[index],0)
             stepBlock = ''
@@ -99,13 +96,13 @@
                     while i < index :
                         block += caseStatements[i]
                         i = i + 1
-                    stepCode[step] = block   
+                    stepCode[step] = block
                     step = step + 1
-                stepBlock= stepBlock + caseStatements[index]
+                stepBlock = stepBlock + caseStatements[index]
                 index = index + 1
                 try :
                     while not re.match('main\.step',caseStatements[index],0) :
-                        stepBlock= stepBlock + caseStatements[index]
+                        stepBlock = stepBlock + caseStatements[index]
                         if index < len(caseStatements)-1:
                             index = index + 1
                         else :
@@ -121,11 +118,10 @@
         if not step_flag :
             stepCode[step] = "".join(caseStatements)
         return stepCode
-    
+
     def getStepCode(self):
         case_step_code = {}
         case_block = self.case_code()
-        
         for case in case_block :
             case_step_code[case] = {}
             step_block = self.step_code(case_block[case])
diff --git a/TestON/core/utilities.py b/TestON/core/utilities.py
index dda43bf..1ba403d 100644
--- a/TestON/core/utilities.py
+++ b/TestON/core/utilities.py
@@ -84,27 +84,27 @@
                 return result
             return assertHandling
 
-    def _assert (self,**assertParam):  
+    def _assert (self,**assertParam):
         '''
         It will take the arguments :
-        expect:'Expected output' 
-        actual:'Actual output' 
+        expect:'Expected output'
+        actual:'Actual output'
         onpass:'Action or string to be triggered or displayed respectively when the assert passed'
         onfail:'Action or string to be triggered or displayed respectively when the assert failed'
         not:'optional argument to specify the negation of the each assertion type'
         operator:'assertion type will be defined by using operator. Like equal , greater, lesser, matches.'
-        
+
         It will return the assertion result.
-                
+
         '''
-              
+
         arguments = self.parse_args(["EXPECT","ACTUAL","ONPASS","ONFAIL","NOT","OPERATOR"],**assertParam)
-        
+
         result = 0
         valuetype = ''
         operation = "not "+ str(arguments["OPERATOR"]) if arguments['NOT'] and arguments['NOT'] == 1 else arguments["OPERATOR"]
         operators = {'equals':{'STR':'==','NUM':'=='}, 'matches' : '=~', 'greater':'>' ,'lesser':'<'}
-           
+
         expectMatch = re.match('^\s*[+-]?0(e0)?\s*$', str(arguments["EXPECT"]), re.I+re.M)
         if not ((not expectMatch) and (arguments["EXPECT"]==0)):
             valuetype = 'NUM'
@@ -112,31 +112,30 @@
             if arguments["OPERATOR"] == 'greater' or arguments["OPERATOR"] == 'lesser':
                 main.log.error("Numeric comparison on strings is not possibele")
                 return main.ERROR
-            
+
         valuetype = 'STR'
         arguments["ACTUAL"] = str(arguments["ACTUAL"])
         if arguments["OPERATOR"] != 'matches':
             arguments["EXPECT"] = str(arguments["EXPECT"])
- 
+
         try :
             opcode = operators[str(arguments["OPERATOR"])][valuetype] if arguments["OPERATOR"] == 'equals' else operators[str(arguments["OPERATOR"])]
-            
+
         except KeyError:
             print "Key Error in assertion"
             return main.FALSE
-        
+
         if opcode == '=~':
             try:
                 assert re.search(str(arguments["EXPECT"]),str(arguments["ACTUAL"]))
                 result = main.TRUE
             except AssertionError:
                 try :
-                    assert re.match(str(arguments["EXPECT"]),str(arguments["ACTUAL"])) 
+                    assert re.match(str(arguments["EXPECT"]),str(arguments["ACTUAL"]))
                     result = main.TRUE
                 except AssertionError:
                     main.log.error("Assertion Failed")
                     result = main.FALSE
-                    
         else :
             try:
                 if str(opcode)=="==":
@@ -145,27 +144,21 @@
                         result = main.TRUE
                     else :
                         result = main.FALSE
-                        
                 elif str(opcode) == ">":
                     main.log.info("Verifying the Expected is Greater than the actual or not using assert_greater")
                     if (ast.literal_eval(arguments["EXPECT"]) > ast.literal_eval(arguments["ACTUAL"])) :
                         result = main.TRUE
                     else :
                         result = main.FALSE
-                        
                 elif str(opcode) == "<":
                     main.log.info("Verifying the Expected is Lesser than the actual or not using assert_lesser")
                     if (ast.literal_eval(arguments["EXPECT"]) < ast.literal_eval(arguments["ACTUAL"])):
                         result = main.TRUE
                     else :
                         result = main.FALSE
-                    
-                    
             except AssertionError:
                 main.log.error("Assertion Failed")
                 result = main.FALSE
-                
-        
         result = result if result else 0
         result = not result if arguments["NOT"] and arguments["NOT"] == 1 else result
         resultString = ""
@@ -179,7 +172,8 @@
             else :
                 main.log.error(arguments["ONFAIL"])
                 main.log.report(arguments["ONFAIL"])
-             
+                main.onFailMsg = arguments[ 'ONFAIL' ]
+
         msg = arguments["ON" + str(resultString)]
 
         if not isinstance(msg,str):
@@ -190,8 +184,7 @@
 
         main.last_result = result
         return result
-    
-    
+
     def parse_args(self,args, **kwargs):
         '''
         It will accept the (key,value) pair and will return the (key,value) pairs with keys in uppercase.
diff --git a/TestON/dependencies/Jenkins_getresult_HA.py b/TestON/dependencies/Jenkins_getresult_HA.py
index 7b4fc09..181e1a9 100755
--- a/TestON/dependencies/Jenkins_getresult_HA.py
+++ b/TestON/dependencies/Jenkins_getresult_HA.py
@@ -6,6 +6,7 @@
 import datetime
 import time
 import argparse
+import glob
 import shutil
 
 parser = argparse.ArgumentParser()
@@ -43,10 +44,10 @@
     ##Graphs
     graphs += '<ac:structured-macro ac:name="html">\n'
     graphs += '<ac:plain-text-body><![CDATA[\n'
-    graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/getPlot?index=2&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
-    graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/getPlot?index=1&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
-    graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/getPlot?index=0&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
-    graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/getPlot?index=3&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
+    graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/Plot-HA/getPlot?index=2&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
+    graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/Plot-HA/getPlot?index=1&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
+    graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/Plot-HA/getPlot?index=0&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
+    graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/Plot-HA/getPlot?index=3&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
     graphs += ']]></ac:plain-text-body>\n'
     graphs += '</ac:structured-macro>\n'
     header +="<p> <a href='https://wiki.onosproject.org/display/OST/Test+Plan+-+HA'>Test Plan for HA Test Cases</a></p>"
@@ -63,7 +64,8 @@
     path = "/home/admin/ONLabTest/TestON/logs/" + name + "/"
     try:
         #IF exists, move the csv file to the workspace
-        shutil.copy(path + test + ".csv", workspace)
+        for csvFile in glob.glob( path + '*.csv' ):
+            shutil.copy( csvFile, workspace )
     except IOError:
         #File probably doesn't exist
         pass
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
index 42fec2d..7136ac6 100755
--- a/TestON/dependencies/rotate.sh
+++ b/TestON/dependencies/rotate.sh
@@ -49,6 +49,6 @@
 root_dir="/home/admin/packet_captures"
 timestamp=`date +%Y_%B_%d_%H_%M_%S`
 #Maybe this should be an argument? pack-and-rotate supports that
-nr_max=20
+nr_max=10
 
 pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap ${root_dir}/${base_name}*.log*" ${nr_max}
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index ce6e02d..57d8c9b 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -252,17 +252,16 @@
            Optional parameter timeout allows you to specify how long to
            wait for pingall to complete
            Optional:
-           timeout(seconds) - This is the pexpect timeout; The function will
-                              timeout if the amount of time between failed
-                              pings exceedes this time and pingall is still
-                              running
+           timeout(seconds) - How long to wait before breaking the pingall
            shortCircuit - Break the pingall based on the number of failed hosts
                           ping
            acceptableFailed - Set the number of acceptable failed pings for the
                               function to still return main.TRUE
            Returns:
            main.TRUE if pingall completes with no pings dropped
-           otherwise main.FALSE"""
+           otherwise main.FALSE
+        """
+        import time
         try:
             timeout = int( timeout )
             if self.handle:
@@ -273,6 +272,7 @@
                 failedPings = 0
                 returnValue = main.TRUE
                 self.handle.sendline( "pingall" )
+                startTime = time.time()
                 while True:
                     i = self.handle.expect( [ "mininet>","X",
                                               pexpect.EOF,
@@ -293,6 +293,12 @@
                                                 + str( failedPings ) +
                                                 " pings failed" )
                                 break
+                        if ( time.time() - startTime ) > timeout:
+                            returnValue = main.FALSE
+                            main.log.error( self.name +
+                                            ": Aborting pingall - " +
+                                            "Function took too long " )
+                            break
                     elif i == 2:
                         main.log.error( self.name +
                                         ": EOF exception found" )
@@ -361,6 +367,64 @@
         main.log.info( self.name + ": \n---\n" + response )
         return main.FALSE
 
+    def pingallHosts( self, hostList, pingType='ipv4' ):
+        """
+            Ping all specified hosts with a specific ping type 
+            
+            Acceptable pingTypes: 
+                - 'ipv4' 
+                - 'ipv6'
+        
+            Acceptable hostList:
+                - ['h1','h2','h3','h4']
+                
+            Returns main.TRUE if all hosts specified can reach 
+            each other
+            
+            Returns main.FALSE if one or more of hosts specified
+            cannot reach each other"""
+            
+        if pingType == "ipv4":
+            cmd = " ping -c 1 -i 1 -W 8 " 
+        elif pingType == "ipv6":
+            cmd = " ping6 -c 1 -i 1 -W 8 "
+        else:
+            main.log.warn( "Invalid pingType specified" )
+            return
+
+        try:
+            main.log.info( "Testing reachability between specified hosts" )
+           
+            isReachable = main.TRUE
+
+            for host in hostList:
+                listIndex = hostList.index(host)
+                # List of hosts to ping other than itself
+                pingList = hostList[:listIndex] + hostList[(listIndex+1):]
+                
+                for temp in pingList:
+                    # Current host pings all other hosts specified
+                    pingCmd = str(host) + cmd + str(temp) 
+                    self.handle.sendline( pingCmd )
+                    i = self.handle.expect( [ pingCmd, pexpect.TIMEOUT ] )
+                    j = self.handle.expect( [ "mininet>", pexpect.TIMEOUT ] )
+                    response = self.handle.before
+                    if re.search( ',\s0\%\spacket\sloss', response ):
+                        main.log.info( str(host) + " -> " + str(temp) )
+                    else:
+                        main.log.info( str(host) + " -> X ("+str(temp)+") "
+                                       " Destination Unreachable" ) 
+                        # One of the host to host pair is unreachable
+                        isReachable = main.FALSE
+
+            return isReachable 
+
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":     " + self.handle.before )
+            main.cleanup()
+            main.exit()
+
     def pingHost( self, **pingParams ):
         """
            Ping from one mininet host to another
@@ -1304,11 +1368,11 @@
         self.handle.sendline('')
         i = self.handle.expect( [ 'mininet>', pexpect.EOF, pexpect.TIMEOUT ],
                                 timeout=2)
+        response = main.TRUE
         if i == 0:
-            self.stopNet()
+            response = self.stopNet()
         elif i == 1:
             return main.TRUE
-        response = main.TRUE
         # print "Disconnecting Mininet"
         if self.handle:
             self.handle.sendline( "exit" )
@@ -1371,20 +1435,32 @@
             response = main.FALSE
         return response
 
-    def arping( self, src, dest, destmac ):
-        self.handle.sendline( '' )
-        self.handle.expect( [ "mininet", pexpect.EOF, pexpect.TIMEOUT ] )
-
-        self.handle.sendline( src + ' arping ' + dest )
+    def arping( self, host="", ip="10.128.20.211", ethDevice="" ):
+        """
+        Description:
+            Sends arp message from mininet host for hosts discovery
+        Required:
+            host - hosts name
+        Optional:
+            ip - ip address that does not exist in the network so there would
+                 be no reply.
+        """
+        if ethDevice:
+            ethDevice = '-I ' + ethDevice + ' '
+        cmd = " py " + host  + ".cmd(\"arping -c 1 " + ethDevice + ip + "\")"
         try:
-            self.handle.expect( [ destmac, pexpect.EOF, pexpect.TIMEOUT ] )
-            main.log.info( self.name + ": ARP successful" )
-            self.handle.expect( [ "mininet", pexpect.EOF, pexpect.TIMEOUT ] )
+            main.log.warn( "Sending: " + cmd )
+            self.handle.sendline( cmd )
+            response = self.handle.before
+            self.handle.sendline( "" )
+            self.handle.expect( "mininet>" )
             return main.TRUE
-        except Exception:
-            main.log.warn( self.name + ": ARP FAILURE" )
-            self.handle.expect( [ "mininet", pexpect.EOF, pexpect.TIMEOUT ] )
-            return main.FALSE
+
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":     " + self.handle.before )
+            main.cleanup()
+            main.exit()
 
     def decToHex( self, num ):
         return hex( num ).split( 'x' )[ 1 ]
@@ -1551,7 +1627,7 @@
         if switchesJson == "":  # if rest call fails
             main.log.error(
                 self.name +
-                ".compare_switches(): Empty JSON object given from ONOS" )
+                ".compareSwitches(): Empty JSON object given from ONOS" )
             return main.FALSE
         onos = switchesJson
         onosDPIDs = []
@@ -1860,7 +1936,7 @@
                     if onosMAC == mnIntf[ 'hw_addr' ].lower() :
                         match = True
                         for ip in mnIntf[ 'ips' ]:
-                            if ip in onosHost[ 'ips' ]:
+                            if ip in onosHost[ 'ipAddresses' ]:
                                 pass  # all is well
                             else:
                                 # misssing ip
@@ -1905,6 +1981,7 @@
         hostStr = handlePy.replace( "]", "" )
         hostStr = hostStr.replace( "'", "" )
         hostStr = hostStr.replace( "[", "" )
+        hostStr = hostStr.replace( " ", "" )
         hostList = hostStr.split( "," )
 
         return hostList
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index e0a205d..c2c011a 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -91,6 +91,7 @@
         elif re.search( "found multiple mininet", outputs ):
             return main.ERROR
         else:
+            # TODO: Parse for failed pings, give some truncated output
             main.log.error( "Error, unexpected output in the ping file" )
             main.log.warn( outputs )
             return main.TRUE
@@ -409,7 +410,6 @@
                 cmd = "sudo -E python opticalTest.py " + controller
                 main.log.info( self.name + ": cmd = " + cmd )
                 self.handle.sendline( cmd )
-                #self.handle.expect( "Press ENTER to push Topology.json" )
                 time.sleep(30)
                 self.handle.sendline( "" )
                 self.handle.sendline( "" )
@@ -448,8 +448,8 @@
             # Close the ssh connection
             self.handle.sendline( "" )
             # self.handle.expect( "\$" )
-            i = self.handle.expect( [ '\$', 'mininet>', pexpect.TIMEOUT ],
-                                    timeout=2)
+            i = self.handle.expect( [ '\$', 'mininet>', pexpect.TIMEOUT,
+                                      pexpect.EOF ], timeout=2 )
             if i == 0:
                 self.handle.sendline( "exit" )
                 self.handle.expect( "closed" )
@@ -460,7 +460,6 @@
                 self.handle.sendline( "exit" )
                 self.handle.expect( "exit" )
                 self.handle.expect( "closed" )
-                
         else:
             main.log.error( "Connection failed to the host" )
         return main.TRUE
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 703e15f..5207f1e 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -188,7 +188,8 @@
             main.cleanup()
             main.exit()
 
-    def startOnosCli( self, ONOSIp, karafTimeout="" ):
+    def startOnosCli( self, ONOSIp, karafTimeout="",
+            commandlineTimeout=10, onosStartTimeout=60 ):
         """
         karafTimeout is an optional argument. karafTimeout value passed
         by user would be used to set the current karaf shell idle timeout.
@@ -206,7 +207,7 @@
         try:
             self.handle.sendline( "" )
             x = self.handle.expect( [
-                "\$", "onos>" ], timeout=10 )
+                "\$", "onos>" ], commandlineTimeout)
 
             if x == 1:
                 main.log.info( "ONOS cli is already running" )
@@ -216,7 +217,7 @@
             self.handle.sendline( "onos -w " + str( ONOSIp ) )
             i = self.handle.expect( [
                 "onos>",
-                pexpect.TIMEOUT ], timeout=60 )
+                pexpect.TIMEOUT ], onosStartTimeout )
 
             if i == 0:
                 main.log.info( str( ONOSIp ) + " CLI Started successfully" )
@@ -280,15 +281,23 @@
                 lvlStr = "--level=" + level
 
             self.handle.sendline( "" )
-            self.handle.expect( "onos>" )
+            i = self.handle.expect( [ "onos>", pexpect.TIMEOUT ] )
+            # TODO: look for bash prompt as well
+            if i == 1:
+                self.handle.sendline( "" )
+                self.handle.expect( "onos>" )
             self.handle.sendline( "log:log " + lvlStr + " " + cmdStr )
+            self.handle.expect( "log:log" )
             self.handle.expect( "onos>" )
 
             response = self.handle.before
             if re.search( "Error", response ):
                 return main.FALSE
             return main.TRUE
-
+        except pexpect.TIMEOUT:
+            main.log.exception( self.name + ": TIMEOUT exception found" )
+            main.cleanup()
+            main.exit()
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
             main.log.error( self.name + ":    " + self.handle.before )
@@ -327,34 +336,35 @@
             main.log.info( "Command '" + str( cmdStr ) + "' sent to "
                            + self.name + "." )
             if debug:
-                main.log.debug( "Raw output" )
-                main.log.debug( repr( response ) )
+                main.log.debug( self.name + ": Raw output" )
+                main.log.debug( self.name + ": " + repr( response ) )
 
             # Remove ANSI color control strings from output
             ansiEscape = re.compile( r'\x1b[^m]*m' )
             response = ansiEscape.sub( '', response )
             if debug:
-                main.log.debug( "ansiEscape output" )
-                main.log.debug( repr( response ) )
+                main.log.debug( self.name + ": ansiEscape output" )
+                main.log.debug( self.name + ": " + repr( response ) )
 
             # Remove extra return chars that get added
             response = re.sub(  r"\s\r", "", response )
             if debug:
-                main.log.debug( "Removed extra returns from output" )
-                main.log.debug( repr( response ) )
+                main.log.debug( self.name + ": Removed extra returns " +
+                                "from output" )
+                main.log.debug( self.name + ": " + repr( response ) )
 
             # Strip excess whitespace
             response = response.strip()
             if debug:
-                main.log.debug( "parsed and stripped output" )
-                main.log.debug( repr( response ) )
+                main.log.debug( self.name + ": parsed and stripped output" )
+                main.log.debug( self.name + ": " + repr( response ) )
 
             # parse for just the output, remove the cmd from response
             output = response.split( cmdStr.strip(), 1 )
             if debug:
-                main.log.debug( "split output" )
+                main.log.debug( self.name + ": split output" )
                 for r in output:
-                    main.log.debug( repr( r ) )
+                    main.log.debug( self.name + ": " + repr( r ) )
             return output[1].strip()
         except IndexError:
             main.log.exception( self.name + ": Object not as expected" )
@@ -471,7 +481,7 @@
     def topology( self ):
         """
         Definition:
-            Returns the ouput of topology command.
+            Returns the output of topology command.
         Return:
             topology = current ONOS topology
         """
@@ -1723,7 +1733,7 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanup()
             main.exit()
-    
+
     def checkIntentState( self, intentsId, expectedState = 'INSTALLED' ):
         """
         Description:
@@ -1740,7 +1750,7 @@
         try:
             # Generating a dictionary: intent id as a key and state as value
             intentsDict = self.getIntentState( intentsId )
-            print "len of intentsDict ", str( len( intentsDict ) )
+            #print "len of intentsDict ", str( len( intentsDict ) )
             if len( intentsId ) != len( intentsDict ):
                 main.log.info( self.name + "There is something wrong " +
                                "getting intents state" )
@@ -1812,15 +1822,18 @@
         """
         try:
             tempFlows = json.loads( self.flows() )
+            #print tempFlows[0]
             returnValue = main.TRUE
+
             for device in tempFlows:
                 for flow in device.get( 'flows' ):
                     if flow.get( 'state' ) != 'ADDED' and flow.get( 'state' ) != \
                             'PENDING_ADD':
                         main.log.info( self.name + ": flow Id: " +
-                                       flow.get( 'flowId' ) +
+                                       flow.get( 'groupId' ) +
                                        " | state:" + flow.get( 'state' ) )
                         returnValue = main.FALSE
+
             return returnValue
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
@@ -2167,7 +2180,7 @@
             elif logLevel == "warn":
                 main.log.warn( output )
             else:
-                main.log.info( output )
+                main.log.info( self.name + ": " + output )
             return result
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
@@ -2285,8 +2298,8 @@
                 # TODO: Should this be main.ERROR?
                 return main.FALSE
             else:
-                main.log.error( "Error in election_test_leader: " +
-                                "unexpected response" )
+                main.log.error( "Error in electionTestLeader on " + self.name +
+                                ": " + "unexpected response" )
                 main.log.error( repr( response ) )
                 return main.FALSE
         except TypeError:
@@ -2326,8 +2339,8 @@
                 main.log.error( "Election app is not loaded on " + self.name )
                 return main.FALSE
             else:
-                main.log.error( "Error in election_test_run: " +
-                                "unexpected response" )
+                main.log.error( "Error in electionTestRun on " + self.name +
+                                ": " + "unexpected response" )
                 main.log.error( repr( response ) )
                 return main.FALSE
         except TypeError:
@@ -2367,8 +2380,8 @@
                 main.log.error( "Election app is not loaded on " + self.name )
                 return main.FALSE
             else:
-                main.log.error( "Error in election_test_withdraw: " +
-                                "unexpected response" )
+                main.log.error( "Error in electionTestWithdraw on " +
+                                self.name + ": " + "unexpected response" )
                 main.log.error( repr( response ) )
                 return main.FALSE
         except TypeError:
@@ -2931,8 +2944,21 @@
                  main.ERROR if there is some error in processing the test
         """
         try:
-            ids = json.loads( self.appIDs( jsonFormat=True ) )
-            apps = json.loads( self.apps( jsonFormat=True ) )
+            bail = False
+            ids = self.appIDs( jsonFormat=True )
+            if ids:
+                ids = json.loads( ids )
+            else:
+                main.log.error( "app-ids returned nothing:" + repr( ids ) )
+                bail = True
+            apps = self.apps( jsonFormat=True )
+            if apps:
+                apps = json.loads( apps )
+            else:
+                main.log.error( "apps returned nothing:" + repr( apps ) )
+                bail = True
+            if bail:
+                return main.FALSE
             result = main.TRUE
             for app in apps:
                 appID = app.get( 'id' )
@@ -2944,7 +2970,7 @@
                     main.log.error( "Error parsing app: " + str( app ) )
                     result = main.FALSE
                 # get the entry in ids that has the same appID
-                current = filter(lambda item: item[ 'id' ] == appID, ids)
+                current = filter( lambda item: item[ 'id' ] == appID, ids )
                 # main.log.debug( "Comparing " + str( app ) + " to " +
                 #                 str( current ) )
                 if not current:  # if ids doesn't have this id
@@ -3044,10 +3070,11 @@
     def setCfg( self, component, propName, value=None, check=True ):
         """
         Set/Unset configuration settings from ONOS cli
-        Optional arguments:
+        Required arguments:
             component - The case sensitive name of the component whose
                         property is to be set
             propName - The case sensitive name of the property to be set/unset
+        Optional arguments:
             value - The value to set the property to. If None, will unset the
                     property and revert it to it's default value(if applicable)
             check - Boolean, Check whether the option was successfully set this
@@ -3096,3 +3123,428 @@
             main.cleanup()
             main.exit()
 
+    def setTestAdd( self, setName, values ):
+        """
+        CLI command to add elements to a distributed set.
+        Arguments:
+            setName - The name of the set to add to.
+            values - The value(s) to add to the set, space seperated.
+        Example usages:
+            setTestAdd( "set1", "a b c" )
+            setTestAdd( "set2", "1" )
+        returns:
+            main.TRUE on success OR
+            main.FALSE if elements were already in the set OR
+            main.ERROR on error
+        """
+        try:
+            cmdStr = "set-test-add " + str( setName ) + " " + str( values )
+            output = self.sendline( cmdStr )
+            try:
+                # TODO: Maybe make this less hardcoded
+                # ConsistentMap Exceptions
+                assert "org.onosproject.store.service" not in output
+                # Node not leader
+                assert "java.lang.IllegalStateException" not in output
+            except AssertionError:
+                main.log.error( "Error in processing 'set-test-add' " +
+                                "command: " + str( output ) )
+                retryTime = 30  # Conservative time, given by Madan
+                main.log.info( "Waiting " + str( retryTime ) +
+                               "seconds before retrying." )
+                time.sleep( retryTime )  # Due to change in mastership
+                output = self.sendline( cmdStr )
+            assert "Error executing command" not in output
+            positiveMatch = "\[(.*)\] was added to the set " + str( setName )
+            negativeMatch = "\[(.*)\] was already in set " + str( setName )
+            main.log.info( self.name + ": " + output )
+            if re.search( positiveMatch, output):
+                return main.TRUE
+            elif re.search( negativeMatch, output):
+                return main.FALSE
+            else:
+                main.log.error( self.name + ": setTestAdd did not" +
+                                " match expected output" )
+                main.log.debug( self.name + " actual: " + repr( output ) )
+                return main.ERROR
+        except AssertionError:
+            main.log.error( "Error in processing 'set-test-add' command: " +
+                            str( output ) )
+            return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def setTestRemove( self, setName, values, clear=False, retain=False ):
+        """
+        CLI command to remove elements from a distributed set.
+        Required arguments:
+            setName - The name of the set to remove from.
+            values - The value(s) to remove from the set, space seperated.
+        Optional arguments:
+            clear - Clear all elements from the set
+            retain - Retain only the  given values. (intersection of the
+                     original set and the given set)
+        returns:
+            main.TRUE on success OR
+            main.FALSE if the set was not changed OR
+            main.ERROR on error
+        """
+        try:
+            cmdStr = "set-test-remove "
+            if clear:
+                cmdStr += "-c " + str( setName )
+            elif retain:
+                cmdStr += "-r " + str( setName ) + " " + str( values )
+            else:
+                cmdStr += str( setName ) + " " + str( values )
+            output = self.sendline( cmdStr )
+            try:
+                # TODO: Maybe make this less hardcoded
+                # ConsistentMap Exceptions
+                assert "org.onosproject.store.service" not in output
+                # Node not leader
+                assert "java.lang.IllegalStateException" not in output
+            except AssertionError:
+                main.log.error( "Error in processing 'set-test-add' " +
+                                "command: " + str( output ) )
+                retryTime = 30  # Conservative time, given by Madan
+                main.log.info( "Waiting " + str( retryTime ) +
+                               "seconds before retrying." )
+                time.sleep( retryTime )  # Due to change in mastership
+                output = self.sendline( cmdStr )
+            assert "Error executing command" not in output
+            main.log.info( self.name + ": " + output )
+            if clear:
+                pattern = "Set " + str( setName ) + " cleared"
+                if re.search( pattern, output ):
+                    return main.TRUE
+            elif retain:
+                positivePattern = str( setName ) + " was pruned to contain " +\
+                                  "only elements of set \[(.*)\]"
+                negativePattern = str( setName ) + " was not changed by " +\
+                                  "retaining only elements of the set " +\
+                                  "\[(.*)\]"
+                if re.search( positivePattern, output ):
+                    return main.TRUE
+                elif re.search( negativePattern, output ):
+                    return main.FALSE
+            else:
+                positivePattern = "\[(.*)\] was removed from the set " +\
+                                  str( setName )
+                if ( len( values.split() ) == 1 ):
+                    negativePattern = "\[(.*)\] was not in set " +\
+                                      str( setName )
+                else:
+                    negativePattern = "No element of \[(.*)\] was in set " +\
+                                      str( setName )
+                if re.search( positivePattern, output ):
+                    return main.TRUE
+                elif re.search( negativePattern, output ):
+                    return main.FALSE
+            main.log.error( self.name + ": setTestRemove did not" +
+                            " match expected output" )
+            main.log.debug( self.name + " expected: " + pattern )
+            main.log.debug( self.name + " actual: " + repr( output ) )
+            return main.ERROR
+        except AssertionError:
+            main.log.error( "Error in processing 'set-test-remove' command: " +
+                            str( output ) )
+            return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def setTestGet( self, setName, values="" ):
+        """
+        CLI command to get the elements in a distributed set.
+        Required arguments:
+            setName - The name of the set to remove from.
+        Optional arguments:
+            values - The value(s) to check if in the set, space seperated.
+        returns:
+            main.ERROR on error OR
+            A list of elements in the set if no optional arguments are
+                supplied OR
+            A tuple containing the list then:
+                main.FALSE if the given values are not in the set OR
+                main.TRUE if the given values are in the set OR
+        """
+        try:
+            values = str( values ).strip()
+            setName = str( setName ).strip()
+            length = len( values.split() )
+            containsCheck = None
+            # Patterns to match
+            setPattern = "\[(.*)\]"
+            pattern = "Items in set " + setName + ":\n" + setPattern
+            containsTrue = "Set " + setName + " contains the value " + values
+            containsFalse = "Set " + setName + " did not contain the value " +\
+                            values
+            containsAllTrue = "Set " + setName + " contains the the subset " +\
+                              setPattern
+            containsAllFalse = "Set " + setName + " did not contain the the" +\
+                               " subset " + setPattern
+
+            cmdStr = "set-test-get "
+            cmdStr += setName + " " + values
+            output = self.sendline( cmdStr )
+            try:
+                # TODO: Maybe make this less hardcoded
+                # ConsistentMap Exceptions
+                assert "org.onosproject.store.service" not in output
+                # Node not leader
+                assert "java.lang.IllegalStateException" not in output
+            except AssertionError:
+                main.log.error( "Error in processing 'set-test-add' " +
+                                "command: " + str( output ) )
+                retryTime = 30  # Conservative time, given by Madan
+                main.log.info( "Waiting " + str( retryTime ) +
+                               "seconds before retrying." )
+                time.sleep( retryTime )  # Due to change in mastership
+                output = self.sendline( cmdStr )
+            assert "Error executing command" not in output
+            main.log.info( self.name + ": " + output )
+
+            if length == 0:
+                match = re.search( pattern, output )
+            else:  # if given values
+                if length == 1:  # Contains output
+                    patternTrue = pattern + "\n" + containsTrue
+                    patternFalse = pattern + "\n" + containsFalse
+                else:  # ContainsAll output
+                    patternTrue = pattern + "\n" + containsAllTrue
+                    patternFalse = pattern + "\n" + containsAllFalse
+                matchTrue = re.search( patternTrue, output )
+                matchFalse = re.search( patternFalse, output )
+                if matchTrue:
+                    containsCheck = main.TRUE
+                    match = matchTrue
+                elif matchFalse:
+                    containsCheck = main.FALSE
+                    match = matchFalse
+                else:
+                    main.log.error( self.name + " setTestGet did not match " +\
+                                    "expected output" )
+                    main.log.debug( self.name + " expected: " + pattern )
+                    main.log.debug( self.name + " actual: " + repr( output ) )
+                    match = None
+            if match:
+                setMatch = match.group( 1 )
+                if setMatch == '':
+                    setList = []
+                else:
+                    setList = setMatch.split( ", " )
+                if length > 0:
+                    return ( setList, containsCheck )
+                else:
+                    return setList
+            else:  # no match
+                main.log.error( self.name + ": setTestGet did not" +
+                                " match expected output" )
+                main.log.debug( self.name + " expected: " + pattern )
+                main.log.debug( self.name + " actual: " + repr( output ) )
+                return main.ERROR
+        except AssertionError:
+            main.log.error( "Error in processing 'set-test-get' command: " +
+                            str( output ) )
+            return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def setTestSize( self, setName ):
+        """
+        CLI command to get the elements in a distributed set.
+        Required arguments:
+            setName - The name of the set to remove from.
+        returns:
+            The integer value of the size returned or
+            None on error
+        """
+        try:
+            # TODO: Should this check against the number of elements returned
+            #       and then return true/false based on that?
+            setName = str( setName ).strip()
+            # Patterns to match
+            setPattern = "\[(.*)\]"
+            pattern = "There are (\d+) items in set " + setName + ":\n" +\
+                          setPattern
+            cmdStr = "set-test-get -s "
+            cmdStr += setName
+            output = self.sendline( cmdStr )
+            try:
+                # TODO: Maybe make this less hardcoded
+                # ConsistentMap Exceptions
+                assert "org.onosproject.store.service" not in output
+                # Node not leader
+                assert "java.lang.IllegalStateException" not in output
+            except AssertionError:
+                main.log.error( "Error in processing 'set-test-add' " +
+                                "command: " + str( output ) )
+                retryTime = 30  # Conservative time, given by Madan
+                main.log.info( "Waiting " + str( retryTime ) +
+                               "seconds before retrying." )
+                time.sleep( retryTime )  # Due to change in mastership
+                output = self.sendline( cmdStr )
+            assert "Error executing command" not in output
+            main.log.info( self.name + ": " + output )
+            match = re.search( pattern, output )
+            if match:
+                setSize = int( match.group( 1 ) )
+                setMatch = match.group( 2 )
+                if len( setMatch.split() ) == setSize:
+                    main.log.info( "The size returned by " + self.name +
+                                   " matches the number of elements in " +
+                                   "the returned set" )
+                else:
+                    main.log.error( "The size returned by " + self.name +
+                                    " does not match the number of " +
+                                    "elements in the returned set." )
+                return setSize
+            else:  # no match
+                main.log.error( self.name + ": setTestGet did not" +
+                                " match expected output" )
+                main.log.debug( self.name + " expected: " + pattern )
+                main.log.debug( self.name + " actual: " + repr( output ) )
+                return None
+        except AssertionError:
+            main.log.error( "Error in processing 'set-test-get' command: " +
+                            str( output ) )
+            return None 
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def counters( self, jsonFormat=True ):
+        """
+        Command to list the various counters in the system.
+        returns:
+            if jsonFormat, a string of the json object returned by the cli
+            command
+            if not jsonFormat, the normal string output of the cli command
+            None on error
+        """
+        try:
+            counters = {}
+            cmdStr = "counters"
+            if jsonFormat:
+                cmdStr += " -j"
+            output = self.sendline( cmdStr )
+            assert "Error executing command" not in output
+            main.log.info( self.name + ": " + output )
+            return output
+        except AssertionError:
+            main.log.error( "Error in processing 'counters' command: " +
+                            str( output ) )
+            return None
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def counterTestIncrement( self, counter, inMemory=False ):
+        """
+        CLI command to increment and get a distributed counter.
+        Required arguments:
+            counter - The name of the counter to increment.
+        Optional arguments:
+            inMemory - use in memory map for the counter
+        returns:
+            integer value of the counter or
+            None on Error
+        """
+        try:
+            counter = str( counter )
+            cmdStr = "counter-test-increment "
+            if inMemory:
+                cmdStr += "-i "
+            cmdStr += counter
+            output = self.sendline( cmdStr )
+            try:
+                # TODO: Maybe make this less hardcoded
+                # ConsistentMap Exceptions
+                assert "org.onosproject.store.service" not in output
+                # Node not leader
+                assert "java.lang.IllegalStateException" not in output
+            except AssertionError:
+                main.log.error( "Error in processing 'set-test-add' " +
+                                "command: " + str( output ) )
+                retryTime = 30  # Conservative time, given by Madan
+                main.log.info( "Waiting " + str( retryTime ) +
+                               "seconds before retrying." )
+                time.sleep( retryTime )  # Due to change in mastership
+                output = self.sendline( cmdStr )
+            assert "Error executing command" not in output
+            main.log.info( self.name + ": " + output )
+            pattern = counter + " was incremented to (\d+)"
+            match = re.search( pattern, output )
+            if match:
+                return int( match.group( 1 ) )
+            else:
+                main.log.error( self.name + ": counterTestIncrement did not" +
+                                " match expected output." )
+                main.log.debug( self.name + " expected: " + pattern )
+                main.log.debug( self.name + " actual: " + repr( output ) )
+                return None
+        except AssertionError:
+            main.log.error( "Error in processing 'counter-test-increment'" +
+                            " command: " + str( output ) )
+            return None
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 7d5ab5e..9d0a416 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -20,6 +20,7 @@
 import time
 import pexpect
 import os.path
+from requests.models import Response
 sys.path.append( "../" )
 from drivers.common.clidriver import CLI
 
@@ -97,7 +98,28 @@
             response = main.FALSE
         return response
 
-    def onosPackage( self ):
+    def getEpochMs( self ):
+        """
+        Returns milliseconds since epoch
+        
+        When checking multiple nodes in a for loop, 
+        around a hundred milliseconds of difference (ascending) is 
+        generally acceptable due to calltime of the function. 
+        Few seconds, however, is not and it means clocks 
+        are off sync. 
+        """
+        try:
+            self.handle.sendline( 'date +%s.%N' )
+            self.handle.expect( 'date \+\%s\.\%N' )
+            self.handle.expect( '\$' )
+            epochMs = self.handle.before
+            return epochMs
+        except Exception:
+            main.log.exception( 'Uncaught exception getting epoch time' )
+            main.cleanup()
+            main.exit()
+
+    def onosPackage( self, opTimeout=30 ):
         """
         Produce a self-contained tar.gz file that can be deployed
         and executed on any platform with Java 7 JRE.
@@ -105,7 +127,7 @@
         try:
             self.handle.sendline( "onos-package" )
             self.handle.expect( "onos-package" )
-            self.handle.expect( "tar.gz", timeout=30 )
+            self.handle.expect( "tar.gz", opTimeout )
             handle = str( self.handle.before )
             main.log.info( "onos-package command returned: " +
                            handle )
@@ -152,7 +174,7 @@
             main.cleanup()
             main.exit()
 
-    def cleanInstall( self ):
+    def cleanInstall( self, mciTimeout=600 ):
         """
         Runs mvn clean install in the root of the ONOS directory.
         This will clean all ONOS artifacts then compile each module
@@ -179,7 +201,7 @@
                     'BUILD\sSUCCESS',
                     'onos\$',  #TODO: fix this to be more generic?
                     'ONOS\$',
-                    pexpect.TIMEOUT ], timeout=600 )
+                    pexpect.TIMEOUT ], mciTimeout )
                 if i == 0:
                     main.log.error( self.name + ":There is insufficient memory \
                             for the Java Runtime Environment to continue." )
@@ -483,6 +505,24 @@
             main.cleanup()
             main.exit()
 
+    def getBranchName( self ):
+        main.log.info( "self.home = " )
+        main.log.info( self.home )
+        self.handle.sendline( "cd " + self.home )
+        self.handle.expect( self.home + "\$" )
+        self.handle.sendline( "git name-rev --name-only HEAD" )
+        self.handle.expect( "git name-rev --name-only HEAD" )
+        self.handle.expect( "\$" )
+
+        lines =  self.handle.before.splitlines()
+        if lines[1] == "master":
+            return "master"
+        elif lines[1] == "onos-1.0":
+            return "onos-1.0"
+        else:
+            main.log.info( lines[1] )
+            return "unexpected ONOS branch for SDN-IP test"
+
     def getVersion( self, report=False ):
         """
         Writes the COMMIT number to the report to be parsed
@@ -508,13 +548,18 @@
             for line in lines:
                 print line
             if report:
+                main.log.wiki( "<blockquote>" )
                 for line in lines[ 2:-1 ]:
                     # Bracket replacement is for Wiki-compliant
                     # formatting. '<' or '>' are interpreted
                     # as xml specific tags that cause errors
                     line = line.replace( "<", "[" )
                     line = line.replace( ">", "]" )
-                    main.log.report( "\t" + line )
+                    #main.log.wiki( "\t" + line )
+                    main.log.wiki( line + "<br /> " )
+                    main.log.summary( line )
+                main.log.wiki( "</blockquote>" )
+                main.log.summary("\n")
             return lines[ 2 ]
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
@@ -562,6 +607,8 @@
         # on here.
         appString = "export ONOS_APPS=" + appString
         mnString = "export OCN="
+        if mnIpAddrs == "":
+            mnString = ""
         onosString = "export OC"
         tempCount = 1
 
@@ -921,7 +968,7 @@
         """
         try:
             self.handle.sendline( "" )
-            self.handle.expect( "\$" )
+            self.handle.expect( "\$", timeout=60 )
             self.handle.sendline( "onos-uninstall " + str( nodeIp ) )
             self.handle.expect( "\$" )
 
@@ -930,6 +977,9 @@
             # onos-uninstall command does not return any text
             return main.TRUE
 
+        except pexpect.TIMEOUT:
+            main.log.exception( self.name + ": Timeout in onosUninstall" )
+            return main.FALSE
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
             main.log.error( self.name + ":    " + self.handle.before )
@@ -1492,13 +1542,18 @@
         except Exception:
             main.log.exception( "Copying files failed" )
 
-    def checkLogs( self, onosIp ):
+    def checkLogs( self, onosIp, restart=False):
         """
         runs onos-check-logs on the given onos node
+        If restart is True, use the old version of onos-check-logs which
+            does not print the full stacktrace, but shows the entire log file,
+            including across restarts
         returns the response
         """
         try:
             cmd = "onos-check-logs " + str( onosIp )
+            if restart:
+                cmd += " old"
             self.handle.sendline( cmd )
             self.handle.expect( cmd )
             self.handle.expect( "\$" )
@@ -1896,3 +1951,300 @@
             main.cleanup()
             main.exit()
 
+    def getOnosIps(self):
+
+        import os
+        
+        # reads env for OC variables, also saves file with OC variables. If file and env conflict 
+        # priority goes to env. If file has OCs that are not in the env, the file OCs are used. 
+        # In other words, if the env is set, the test will use those values. 
+
+        # returns a list of ip addresses for the onos nodes, will work with up to 7 nodes + OCN and OCI
+        # returns in format [ OC1 ip, OC2 ...ect. , OCN, OCI ]
+
+        envONOSIps = {}
+
+        x = 1
+        while True:
+            try:
+                temp = os.environ[ 'OC' + str(x) ]
+            except KeyError: 
+                break
+            envONOSIps[ ("OC" + str(x)) ] = temp 
+            x += 1 
+
+        try: 
+            temp = os.environ[ 'OCN' ] 
+            envONOSIps[ "OCN" ] = temp
+        except KeyError: 
+            main.log.info("OCN not set in env")
+
+        try:
+            temp = os.environ[ 'OCI' ]
+            envONOSIps[ "OCI" ] = temp
+        except:
+            main.log.error("OCI not set in env")
+
+        print(str(envONOSIps))
+
+        order = [ "OC1", "OC2", "OC3","OC4","OC5","OC6","OC7","OCN","OCI" ]
+        ONOSIps = []
+
+        try: 
+            if os.path.exists("myIps"):
+                ipFile = open("myIps","r+")
+            else: 
+                ipFile = open("myIps","w+")
+
+            fileONOSIps = ipFile.readlines()
+            ipFile.close()
+
+            print str(fileONOSIps)
+            
+            if str(fileONOSIps) == "[]": 
+                ipFile = open("myIps","w+")
+                for key in envONOSIps:
+                    ipFile.write(key+ "=" + envONOSIps[key] + "\n")
+                ipFile.close()
+                for i in order: 
+                    if i in envONOSIps: 
+                        ONOSIps.append(envONOSIps[i])
+                
+                return ONOSIps 
+
+            else: 
+                fileDict = {}
+                for line in fileONOSIps: 
+                    line = line.replace("\n","")
+                    line = line.split("=") 
+                    key = line[0]
+                    value = line[1]
+                    fileDict[key] = value 
+
+                for x in envONOSIps: 
+                    if x in fileDict: 
+                        if envONOSIps[x] == fileDict[x]: 
+                            continue
+                        else: 
+                            fileDict[x] = envONOSIps[x]
+                    else: 
+                        fileDict[x] = envONOSIps[x]
+
+                ipFile = open("myIps","w+")
+                for key in order: 
+                    if key in fileDict: 
+                        ipFile.write(key + "=" + fileDict[key] + "\n")
+                        ONOSIps.append(fileDict[key]) 
+                ipFile.close()
+
+                return ONOSIps 
+
+        except IOError as a:
+            main.log.error(a) 
+
+        except Exception as a:
+            main.log.error(a) 
+
+
+    def logReport(self, nodeIp, searchTerms, outputMode="s"):
+        '''
+            - accepts either a list or a string for "searchTerms" these
+              terms will be searched for in the log and have their 
+              instances counted 
+
+            - nodeIp is the ip of the node whos log is to be scanned 
+
+            - output modes: 
+                "s" -   Simple. Quiet output mode that just prints 
+                        the occurences of each search term 
+
+                "d" -   Detailed. Prints number of occurences as well as the entire
+                        line for each of the last 5 occurences 
+
+            - returns total of the number of instances of all search terms
+        '''
+        main.log.info("========================== Log Report ===========================\n")
+
+        if type(searchTerms) is str: 
+            searchTerms = [searchTerms]
+
+        logLines = [ [" "] for i in range(len(searchTerms)) ]
+
+        for term in range(len(searchTerms)): 
+            logLines[term][0] = searchTerms[term]
+
+        totalHits = 0 
+        for term in range(len(searchTerms)): 
+            cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep " + searchTerms[term] 
+            self.handle.sendline(cmd)
+            self.handle.expect(":~")
+            before = (self.handle.before).splitlines()
+
+            count = [searchTerms[term],0]
+
+            for line in before:
+                if searchTerms[term] in line and "grep" not in line:
+                    count[1] += 1 
+                    if before.index(line) > ( len(before) - 7 ):
+                        logLines[term].append(line)
+
+            main.log.info( str(count[0]) + ": " + str(count[1]) )
+            if term == len(searchTerms)-1: 
+                print("\n")
+            totalHits += int(count[1]) 
+
+        if outputMode != "s" and outputMode != "S":        
+            outputString = ""
+            for i in logLines:
+                outputString = i[0] + ": \n" 
+                for x in range(1,len(i)): 
+                    outputString += ( i[x] + "\n" )    
+                
+                if outputString != (i[0] + ": \n"):
+                    main.log.info(outputString) 
+                
+        main.log.info("================================================================\n")
+        return totalHits 
+
+    def getOnosIpFromEnv(self):
+
+        import string  
+
+        # returns a list of ip addresses for the onos nodes, will work with up to 7 nodes + OCN and OCI
+        # returns in format [ 'x', OC1 ip, OC2 i... ect. ... , ONN ip ]
+
+        self.handle.sendline("env| grep OC") 
+        self.handle.expect(":~")
+        rawOutput = self.handle.before
+        print rawOutput
+        print "-----------------------------"
+        print repr(rawOutput)
+        mpa = dict.fromkeys(range(32))
+        translated = rawOutput.translate(mpa)
+        print translated
+
+
+        # create list with only the lines that have the needed IPs 
+        unparsedIps = []
+
+        # remove preceeding or trailing lines
+        for line in rawOutput: 
+            if "OC" in line and "=" in line: 
+                unparsedIps.append(str(line)) 
+
+        # determine cluster size
+        clusterCount = 0
+        for line in unparsedIps:
+            line = str(line)
+            print line
+            temp = line.replace("OC","")
+            print("line index " + str(line.index("="))) 
+            OCindex = temp[0]
+            for i in range(0, 7):
+                if OCindex == str(i) and i > clusterCount:
+                    clusterCount == i
+                    print(clusterCount)
+        # create list to hold ips such that OC1 is at list[1] and OCN and OCI are at the end (in that order)
+        ONOSIps = ["x"] * (clusterCount + 3) 
+
+        # populate list 
+        for line in unparsedIps:
+            main.log.info(line)########## 
+            temp = str(line.replace("OC",""))
+            main.log.info(str(list(temp)))
+            OCindex = temp[0]
+            main.log.info(OCindex)############
+            if OCindex == "N":
+                ONOSIps[ clusterCount + 1 ] = temp.replace("N=","")
+    
+            if OCindex == "I":
+                ONOSIps[ clusterCount + 2 ] = temp.replace("I=","")
+
+            else:
+                ONOSIps[ int(OCindex) ] = temp.replace((OCindex + "=") ,"")
+
+        # validate 
+        for x in ONOSIps: 
+            if ONOSIps.index(x) != 0 and x == "x": 
+                main.log.error("ENV READ FAILURE, MISSING DATA: \n\n" + str(ONOSIps) + "\n\n") 
+
+        return ONOSIps
+
+
+    def onosErrorLog(self, nodeIp): 
+
+        cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep WARN"
+        self.handle.sendline(cmd) 
+        self.handle.expect(":~")
+        before = (self.handle.before).splitlines() 
+
+        warnings = []
+
+        for line in before: 
+            if "WARN" in line and "grep" not in line: 
+                warnings.append(line) 
+                main.warnings[main.warnings[0]+1] = line
+                main.warnings[0] += 1
+                if main.warnings[0] >= 10: 
+                    main.warnings[0] = 0 
+
+        cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep ERROR"
+        self.handle.sendline(cmd)
+        self.handle.expect(":~")
+        before = (self.handle.before).splitlines()
+
+        errors = []
+
+        for line in before:
+            if "ERROR" in line and "grep" not in line:
+                errors.append(line)
+                main.errors[main.errors[0]+1] = line
+                main.errors[0] += 1
+                if main.errors[0] >= 10:
+                    main.errors[0] = 0
+
+        cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep Exept"
+        self.handle.sendline(cmd)
+        self.handle.expect(":~")
+        before = (self.handle.before).splitlines()
+
+        exceptions = []
+
+        for line in before:
+            if "Except" in line and "grep" not in line:
+                exceptions.append(line)
+                main.exceptions[main.errors[0]+1] = line
+                main.exceptions[0] += 1
+                if main.exceptions[0] >= 10:
+                    main.exceptions[0] = 0
+
+
+
+        ################################################################
+
+        msg1 = "WARNINGS: \n"
+        for i in main.warnings: 
+            if type(i) is not int: 
+                msg1 += ( i + "\n")
+
+        msg2 = "ERRORS: \n"
+        for i in main.errors:
+            if type(i) is not int:
+                msg2 += ( i + "\n")
+
+        msg3 = "EXCEPTIONS: \n"
+        for i in main.exceptions: 
+            if type(i) is not int:
+                msg3 += ( i + "\n")
+
+        main.log.info("===============================================================\n") 
+        main.log.info( "Warnings: " + str(len(warnings))) 
+        main.log.info( "Errors: " + str(len(errors))) 
+        main.log.info( "Exceptions: " + str(len(exceptions)))
+        if len(warnings) > 0:
+            main.log.info(msg1)
+        if len(errors) > 0: 
+            main.log.info(msg2)
+        if len(exceptions) > 0: 
+            main.log.info(msg3)
+        main.log.info("===============================================================\n")
diff --git a/TestON/drivers/common/cli/quaggaclidriver.py b/TestON/drivers/common/cli/quaggaclidriver.py
index 1c63206..7e3df2b 100644
--- a/TestON/drivers/common/cli/quaggaclidriver.py
+++ b/TestON/drivers/common/cli/quaggaclidriver.py
@@ -40,9 +40,9 @@
                 ip_address="1.1.1.1",
                 port=self.port,
                 pwd=self.pwd )
-        main.log.info( "connect parameters:" + str( self.user_name ) + ";"
-                       + str( self.ip_address ) + ";" + str( self.port )
-                       + ";" + str(self.pwd ) )
+        #main.log.info( "connect parameters:" + str( self.user_name ) + ";"
+        #               + str( self.ip_address ) + ";" + str( self.port )
+        #               + ";" + str(self.pwd ) )
 
         if self.handle:
             # self.handle.expect( "",timeout=10 )
@@ -186,7 +186,7 @@
         return intents
 
     # This method extracts all actual routes from ONOS CLI
-    def extractActualRoutes( self, getRoutesResult ):
+    def extractActualRoutesOneDotZero( self, getRoutesResult ):
         routesJsonObj = json.loads( getRoutesResult )
 
         allRoutesActual = []
@@ -199,6 +199,18 @@
 
         return sorted( allRoutesActual )
 
+    def extractActualRoutesMaster( self, getRoutesResult ):
+        routesJsonObj = json.loads( getRoutesResult )
+
+        allRoutesActual = []
+        for route in routesJsonObj['routes4']:
+            if route[ 'prefix' ] == '172.16.10.0/24':
+                continue
+            allRoutesActual.append(
+                route[ 'prefix' ] + "/" + route[ 'nextHop' ] )
+
+        return sorted( allRoutesActual )
+
     # This method extracts all actual route intents from ONOS CLI
     def extractActualRouteIntents( self, getIntentsResult ):
         intents = []
@@ -207,8 +219,8 @@
         intentsJsonObj = json.loads( getIntentsResult )
 
         for intent in intentsJsonObj:
-            if intent[ 'appId' ] != "org.onosproject.sdnip":
-                continue
+            #if intent[ 'appId' ] != "org.onosproject.sdnip":
+            #    continue
             if intent[ 'type' ] == "MultiPointToSinglePointIntent" \
             and intent[ 'state' ] == 'INSTALLED':
                 egress = str( intent[ 'egress' ][ 'device' ] ) + ":" \
@@ -230,6 +242,26 @@
                 intents.append( intent )
         return sorted( intents )
 
+    # This method calculates the MultiPointToSinglePointIntent number installed
+    def extractActualRouteIntentNum( self, getIntentsResult ):
+        intentsJsonObj = json.loads( getIntentsResult )
+        num = 0
+        for intent in intentsJsonObj:
+            if intent[ 'type' ] == "MultiPointToSinglePointIntent" \
+            and intent[ 'state' ] == 'INSTALLED':
+                num = num + 1
+        return num
+
+    # This method calculates the PointToPointIntent number installed
+    def extractActualBgpIntentNum( self, getIntentsResult ):
+        intentsJsonObj = json.loads( getIntentsResult )
+        num = 0
+        for intent in intentsJsonObj:
+            if intent[ 'type' ] == "PointToPointIntent" \
+            and intent[ 'state' ] == 'INSTALLED':
+                num = num + 1
+        return num
+
     # This method extracts all actual BGP intents from ONOS CLI
     def extractActualBgpIntents( self, getIntentsResult ):
         intents = []
@@ -238,8 +270,8 @@
         intentsJsonObj = json.loads( getIntentsResult )
 
         for intent in intentsJsonObj:
-            if intent[ 'appId' ] != "org.onosproject.sdnip":
-                continue
+            #if intent[ 'appId' ] != "org.onosproject.sdnip":
+            #    continue
             if intent[ 'type' ] == "PointToPointIntent" \
             and "protocol=6" in str( intent[ 'selector' ] ):
                 ingress = str( intent[ 'ingress' ][ 'device' ] ) + ":" \
@@ -489,7 +521,7 @@
         if routesAdded == numRoutes:
             return main.TRUE
         return main.FALSE
-    
+
     # Please use deleteRoutes method instead of this one!
     def delRoute( self, net, numRoutes, routeRate ):
         try:
@@ -562,7 +594,7 @@
             child.expect( "Flow table show" )
             count = 0
             while True:
-                i = child.expect( [ '17\d\.\d{1,3}\.\d{1,3}\.\d{1,3}', 
+                i = child.expect( [ '17\d\.\d{1,3}\.\d{1,3}\.\d{1,3}',
                                    'CLI#', pexpect.TIMEOUT ] )
                 if i == 0:
                     count = count + 1
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 44552ae..3b94ef7 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -81,7 +81,7 @@
         while i == 5:
             i = self.handle.expect( [
 				    ssh_newkey,
-                                    'password:',
+                                    'password:|Password:',
                                     pexpect.EOF,
                                     pexpect.TIMEOUT,
                                     refused,
diff --git a/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py b/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py
new file mode 100644
index 0000000..3f41180
--- /dev/null
+++ b/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py
@@ -0,0 +1,967 @@
+"""
+    Wrapper functions for FuncIntent
+    This functions include Onosclidriver and Mininetclidriver driver functions
+    Author: kelvin@onlab.us
+"""
+def __init__( self ):
+    self.default = ''
+
+def hostIntent( main,
+                name,
+                host1,
+                host2,
+                host1Id="",
+                host2Id="",
+                mac1="",
+                mac2="",
+                vlan1="-1",
+                vlan2="-1",
+                sw1="",
+                sw2="",
+                expectedLink=0 ):
+    """
+        Description:
+            Verify add-host-intent
+        Steps:
+            - Discover hosts
+            - Add host intents
+            - Check intents
+            - Verify flows
+            - Ping hosts
+            - Reroute
+                - Link down
+                - Verify flows
+                - Check topology
+                - Ping hosts
+                - Link up
+                - Verify flows
+                - Check topology
+                - Ping hosts
+            - Remove intents
+        Required:
+            name - Type of host intent to add eg. IPV4 | VLAN | Dualstack
+            host1 - Name of first host
+            host2 - Name of second host
+        Optional:
+            host1Id - ONOS id of the first host eg. 00:00:00:00:00:01/-1
+            host2Id - ONOS id of the second host
+            mac1 - Mac address of first host
+            mac2 - Mac address of the second host
+            vlan1 - Vlan tag of first host, defaults to -1
+            vlan2 - Vlan tag of second host, defaults to -1
+            sw1 - First switch to bring down & up for rerouting purpose
+            sw2 - Second switch to bring down & up for rerouting purpose
+            expectedLink - Expected link when the switches are down, it should
+                           be two links lower than the links before the two
+                           switches are down
+    """
+    import time
+
+    # Assert variables
+    assert main, "There is no main variable"
+    assert name, "variable name is empty"
+    assert host1 and host2, "You must specify hosts"
+
+    global itemName
+    itemName = name
+    h1Id = host1Id
+    h2Id = host2Id
+    h1Mac = mac1
+    h2Mac = mac2
+    vlan1 = vlan1
+    vlan2 = vlan2
+    hostNames = [ host1 , host2 ]
+    intentsId = []
+    stepResult = main.TRUE
+    pingResult = main.TRUE
+    intentResult = main.TRUE
+    removeIntentResult = main.TRUE
+    flowResult = main.TRUE
+    topoResult = main.TRUE
+    linkDownResult = main.TRUE
+    linkUpResult = main.TRUE
+
+    if main.hostsData:
+        if not h1Mac:
+            h1Mac = main.hostsData[ host1 ][ 'mac' ]
+        if not h2Mac:
+            h2Mac = main.hostsData[ host2 ][ 'mac' ]
+        if main.hostsData[ host1 ][ 'vlan' ] != '-1':
+            vlan1 = main.hostsData[ host1 ][ 'vlan' ]
+        if main.hostsData[ host2 ][ 'vlan' ] != '-1':
+            vlan2 = main.hostsData[ host2 ][ 'vlan' ]
+        if not h1Id:
+            h1Id = main.hostsData[ host1 ][ 'id' ]
+        if not h2Id:
+            h2Id = main.hostsData[ host2 ][ 'id' ]
+
+    assert h1Id and h2Id, "You must specify host IDs"
+    if not ( h1Id and h2Id ):
+        main.log.info( "There are no host IDs" )
+        return main.FALSE
+
+    # Discover hosts using arping
+    main.log.info( itemName + ": Discover host using arping" )
+    main.Mininet1.arping( host=host1 )
+    main.Mininet1.arping( host=host2 )
+    host1 = main.CLIs[ 0 ].getHost( mac=h1Mac )
+    host2 = main.CLIs[ 0 ].getHost( mac=h2Mac )
+
+    # Adding host intents
+    main.log.info( itemName + ": Adding host intents" )
+    intent1 = main.CLIs[ 0 ].addHostIntent( hostIdOne=h1Id,
+                                           hostIdTwo=h2Id )
+    intentsId.append( intent1 )
+    time.sleep( 5 )
+
+    # Check intents state
+    time.sleep( 30 )
+    intentResult = checkIntentState( main, intentsId )
+
+    # Check intents state again if first check fails...
+    if not intentResult:
+        intentResult = checkIntentState( main, intentsId )
+
+    # Verify flows
+    checkFlowsState( main )
+
+    # Ping hosts
+    firstPingResult = pingallHosts( main, hostNames )
+    if not firstPingResult:
+        main.log.debug( "First ping failed, there must be" +
+                       " something wrong with ONOS performance" )
+
+    # Ping hosts again...
+    pingResult = pingResult and pingallHosts( main, hostNames )
+    time.sleep( 5 )
+
+    # Test rerouting if these variables exist
+    if sw1 and sw2 and expectedLink:
+        # link down
+        linkDownResult = link( main, sw1, sw2, "down" )
+        intentResult = intentResult and checkIntentState( main, intentsId )
+
+        # Verify flows
+        checkFlowsState( main )
+
+        # Check OnosTopology
+        topoResult = checkTopology( main, expectedLink )
+
+        # Ping hosts
+        pingResult = pingResult and pingallHosts( main, hostNames )
+
+        intentResult = checkIntentState( main, intentsId )
+
+        # Checks ONOS state in link down
+        if linkDownResult and topoResult and pingResult and intentResult:
+            main.log.info( itemName + ": Successfully brought link down" )
+        else:
+            main.log.error( itemName + ": Failed to bring link down" )
+
+        # link up
+        linkUpResult = link( main, sw1, sw2, "up" )
+        time.sleep( 5 )
+
+        # Verify flows
+        checkFlowsState( main )
+
+        # Check OnosTopology
+        topoResult = checkTopology( main, main.numLinks )
+
+        # Ping hosts
+        pingResult = pingResult and pingallHosts( main, hostNames )
+
+        intentResult = checkIntentState( main, intentsId )
+
+        # Checks ONOS state in link up
+        if linkUpResult and topoResult and pingResult and intentResult:
+            main.log.info( itemName + ": Successfully brought link back up" )
+        else:
+            main.log.error( itemName + ": Failed to bring link back up" )
+
+    # Remove all intents
+    removeIntentResult = removeAllIntents( main, intentsId )
+
+    stepResult = pingResult and linkDownResult and linkUpResult \
+                 and intentResult and removeIntentResult
+
+    return stepResult
+
+def pointIntent( main,
+                 name,
+                 host1,
+                 host2,
+                 deviceId1="",
+                 deviceId2="",
+                 port1="",
+                 port2="",
+                 ethType="",
+                 mac1="",
+                 mac2="",
+                 bandwidth="",
+                 lambdaAlloc=False,
+                 ipProto="",
+                 ip1="",
+                 ip2="",
+                 tcp1="",
+                 tcp2="",
+                 sw1="",
+                 sw2="",
+                 expectedLink=0 ):
+
+    """
+        Description:
+            Verify add-point-intent
+        Steps:
+            - Get device ids | ports
+            - Add point intents
+            - Check intents
+            - Verify flows
+            - Ping hosts
+            - Reroute
+                - Link down
+                - Verify flows
+                - Check topology
+                - Ping hosts
+                - Link up
+                - Verify flows
+                - Check topology
+                - Ping hosts
+            - Remove intents
+        Required:
+            name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+            host1 - Name of first host
+            host2 - Name of second host
+        Optional:
+            deviceId1 - ONOS device id of the first switch, the same as the
+                        location of the first host eg. of:0000000000000001/1,
+                        located at device 1 port 1
+            deviceId2 - ONOS device id of the second switch
+            port1 - The port number where the first host is attached
+            port2 - The port number where the second host is attached
+            ethType - Ethernet type eg. IPV4, IPV6
+            mac1 - Mac address of first host
+            mac2 - Mac address of the second host
+            bandwidth - Bandwidth capacity
+            lambdaAlloc - Allocate lambda, defaults to False
+            ipProto - IP protocol
+            ip1 - IP address of first host
+            ip2 - IP address of second host
+            tcp1 - TCP port of first host
+            tcp2 - TCP port of second host
+            sw1 - First switch to bring down & up for rerouting purpose
+            sw2 - Second switch to bring down & up for rerouting purpose
+            expectedLink - Expected link when the switches are down, it should
+                           be two links lower than the links before the two
+                           switches are down
+    """
+
+    import time
+    assert main, "There is no main variable"
+    assert name, "variable name is empty"
+    assert host1 and host2, "You must specify hosts"
+
+    global itemName
+    itemName = name
+    host1 = host1
+    host2 = host2
+    hostNames = [ host1, host2 ]
+    intentsId = []
+
+    pingResult = main.TRUE
+    intentResult = main.TRUE
+    removeIntentResult = main.TRUE
+    flowResult = main.TRUE
+    topoResult = main.TRUE
+    linkDownResult = main.TRUE
+    linkUpResult = main.TRUE
+
+    # Adding bidirectional point  intents
+    main.log.info( itemName + ": Adding point intents" )
+    intent1 = main.CLIs[ 0 ].addPointIntent( ingressDevice=deviceId1,
+                                             egressDevice=deviceId2,
+                                             portIngress=port1,
+                                             portEgress=port2,
+                                             ethType=ethType,
+                                             ethSrc=mac1,
+                                             ethDst=mac2,
+                                             bandwidth=bandwidth,
+                                             lambdaAlloc=lambdaAlloc,
+                                             ipProto=ipProto,
+                                             ipSrc=ip1,
+                                             ipDst=ip2,
+                                             tcpSrc=tcp1,
+                                             tcpDst=tcp2 )
+
+    intentsId.append( intent1 )
+    time.sleep( 5 )
+    intent2 = main.CLIs[ 0 ].addPointIntent( ingressDevice=deviceId2,
+                                             egressDevice=deviceId1,
+                                             portIngress=port2,
+                                             portEgress=port1,
+                                             ethType=ethType,
+                                             ethSrc=mac2,
+                                             ethDst=mac1,
+                                             bandwidth=bandwidth,
+                                             lambdaAlloc=lambdaAlloc,
+                                             ipProto=ipProto,
+                                             ipSrc=ip2,
+                                             ipDst=ip1,
+                                             tcpSrc=tcp2,
+                                             tcpDst=tcp1 )
+    intentsId.append( intent2 )
+
+    # Check intents state
+    time.sleep( 30 )
+    intentResult = checkIntentState( main, intentsId )
+
+    # Check intents state again if first check fails...
+    if not intentResult:
+        intentResult = checkIntentState( main, intentsId )
+
+    # Verify flows
+    checkFlowsState( main )
+
+    # Ping hosts
+    firstPingResult = pingallHosts( main, hostNames )
+    if not firstPingResult:
+        main.log.debug( "First ping failed, there must be" +
+                       " something wrong with ONOS performance" )
+
+    # Ping hosts again...
+    pingResult = pingResult and pingallHosts( main, hostNames )
+    time.sleep( 5 )
+
+    # Test rerouting if these variables exist
+    if sw1 and sw2 and expectedLink:
+        # link down
+        linkDownResult = link( main, sw1, sw2, "down" )
+        intentResult = intentResult and checkIntentState( main, intentsId )
+
+        # Verify flows
+        checkFlowsState( main )
+
+        # Check OnosTopology
+        topoResult = checkTopology( main, expectedLink )
+
+        # Ping hosts
+        pingResult = pingResult and pingallHosts( main, hostNames )
+
+        intentResult = checkIntentState( main, intentsId )
+
+        # Checks ONOS state in link down
+        if linkDownResult and topoResult and pingResult and intentResult:
+            main.log.info( itemName + ": Successfully brought link down" )
+        else:
+            main.log.error( itemName + ": Failed to bring link down" )
+
+        # link up
+        linkUpResult = link( main, sw1, sw2, "up" )
+        time.sleep( 5 )
+
+        # Verify flows
+        checkFlowsState( main )
+
+        # Check OnosTopology
+        topoResult = checkTopology( main, main.numLinks )
+
+        # Ping hosts
+        pingResult = pingResult and pingallHosts( main, hostNames )
+
+        intentResult = checkIntentState( main, intentsId )
+
+        # Checks ONOS state in link up
+        if linkUpResult and topoResult and pingResult and intentResult:
+            main.log.info( itemName + ": Successfully brought link back up" )
+        else:
+            main.log.error( itemName + ": Failed to bring link back up" )
+
+    # Remove all intents
+    removeIntentResult = removeAllIntents( main, intentsId )
+
+    stepResult = pingResult and linkDownResult and linkUpResult \
+                 and intentResult and removeIntentResult
+
+    return stepResult
+
+def singleToMultiIntent( main,
+                         name,
+                         hostNames,
+                         devices="",
+                         ports=None,
+                         ethType="",
+                         macs=None,
+                         bandwidth="",
+                         lambdaAlloc=False,
+                         ipProto="",
+                         ipAddresses="",
+                         tcp="",
+                         sw1="",
+                         sw2="",
+                         expectedLink=0 ):
+    """
+        Verify Single to Multi Point intents
+        NOTE:If main.hostsData is not defined, variables data should be passed in the
+        same order index wise. All devices in the list should have the same
+        format, either all the devices have its port or it doesn't.
+        eg. hostName = [ 'h1', 'h2' ,..  ]
+            devices = [ 'of:0000000000000001', 'of:0000000000000002', ...]
+            ports = [ '1', '1', ..]
+            ...
+        Description:
+            Verify add-single-to-multi-intent iterates through the list of given
+            host | devices and add intents
+        Steps:
+            - Get device ids | ports
+            - Add single to multi point intents
+            - Check intents
+            - Verify flows
+            - Ping hosts
+            - Reroute
+                - Link down
+                - Verify flows
+                - Check topology
+                - Ping hosts
+                - Link up
+                - Verify flows
+                - Check topology
+                - Ping hosts
+            - Remove intents
+        Required:
+            name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+            hostNames - List of host names
+        Optional:
+            devices - List of device ids in the same order as the hosts
+                      in hostNames
+            ports - List of port numbers in the same order as the device in
+                    devices
+            ethType - Ethernet type eg. IPV4, IPV6
+            macs - List of hosts mac address in the same order as the hosts in
+                   hostNames
+            bandwidth - Bandwidth capacity
+            lambdaAlloc - Allocate lambda, defaults to False
+            ipProto - IP protocol
+            ipAddresses - IP addresses of host in the same order as the hosts in
+                          hostNames
+            tcp - TCP ports in the same order as the hosts in hostNames
+            sw1 - First switch to bring down & up for rerouting purpose
+            sw2 - Second switch to bring down & up for rerouting purpose
+            expectedLink - Expected link when the switches are down, it should
+                           be two links lower than the links before the two
+                           switches are down
+    """
+
+    import time
+    import copy
+    assert main, "There is no main variable"
+    assert hostNames, "You must specify hosts"
+    assert devices or main.hostsData, "You must specify devices"
+
+    global itemName
+    itemName = name
+    tempHostsData = {}
+    intentsId = []
+
+    macsDict = {}
+    ipDict = {}
+    if hostNames and devices:
+        if len( hostNames ) != len( devices ):
+            main.log.debug( "hosts and devices does not have the same length" )
+            #print "len hostNames = ", len( hostNames )
+            #print "len devices = ", len( devices )
+            return main.FALSE
+        if ports:
+            if len( ports ) != len( devices ):
+                main.log.error( "Ports and devices does " +
+                                "not have the same length" )
+                #print "len devices = ", len( devices )
+                #print "len ports = ", len( ports )
+                return main.FALSE
+        for i in range( len( devices ) ):
+            macsDict[ devices[ i ] ] = macs[ i ]
+        else:
+            main.log.info( "Device Ports are not specified" )
+    elif hostNames and not devices and main.hostsData:
+        devices = []
+        main.log.info( "singleToMultiIntent function is using main.hostsData" ) 
+        for host in hostNames:
+               devices.append( main.hostsData.get( host ).get( 'location' ) )
+               macsDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+                           main.hostsData.get( host ).get( 'mac' )
+               ipDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+                           main.hostsData.get( host ).get( 'ipAddresses' )
+        #print main.hostsData
+
+    #print 'host names = ', hostNames
+    #print 'devices = ', devices
+    #print "macsDict = ", macsDict
+
+    pingResult = main.TRUE
+    intentResult = main.TRUE
+    removeIntentResult = main.TRUE
+    flowResult = main.TRUE
+    topoResult = main.TRUE
+    linkDownResult = main.TRUE
+    linkUpResult = main.TRUE
+
+    devicesCopy = copy.copy( devices )
+    if ports:
+        portsCopy = copy.copy( ports )
+    main.log.info( itemName + ": Adding single point to multi point intents" )
+    # Adding bidirectional point  intents
+    for i in range( len( devices ) ):
+        ingressDevice = devicesCopy[ i ]
+        egressDeviceList = copy.copy( devicesCopy )
+        egressDeviceList.remove( ingressDevice )
+        if ports:
+            portIngress = portsCopy[ i ]
+            portEgressList = copy.copy( portsCopy )
+            del portEgressList[ i ]
+        else:
+            portIngress = ""
+            portEgressList = None
+        if not macsDict:
+            srcMac = ""
+        else:
+            srcMac = macsDict[ ingressDevice ]
+            if srcMac == None:
+                main.log.debug( "There is no MAC in device - " + ingressDevice )
+                srcMac = ""
+
+        intentsId.append( main.CLIs[ 0 ].addSinglepointToMultipointIntent(
+                                            ingressDevice=ingressDevice,
+                                            egressDeviceList=egressDeviceList,
+                                            portIngress=portIngress,
+                                            portEgressList=portEgressList,
+                                            ethType=ethType,
+                                            ethSrc=srcMac,
+                                            bandwidth=bandwidth,
+                                            lambdaAlloc=lambdaAlloc,
+                                            ipProto=ipProto,
+                                            ipSrc="",
+                                            ipDst="",
+                                            tcpSrc="",
+                                            tcpDst="" ) )
+
+    pingResult = pingallHosts( main, hostNames )
+
+    # Check intents state
+    time.sleep( 30 )
+    intentResult = checkIntentState( main, intentsId )
+
+    # Check intents state again if first check fails...
+    if not intentResult:
+        intentResult = checkIntentState( main, intentsId )
+
+    # Verify flows
+    checkFlowsState( main )
+
+    # Ping hosts
+    pingResult = pingResult and pingallHosts( main, hostNames )
+    # Ping hosts again...
+    pingResult = pingResult and pingallHosts( main, hostNames )
+    time.sleep( 5 )
+
+    # Test rerouting if these variables exist
+    if sw1 and sw2 and expectedLink:
+        # link down
+        linkDownResult = link( main, sw1, sw2, "down" )
+        intentResult = intentResult and checkIntentState( main, intentsId )
+
+        # Verify flows
+        checkFlowsState( main )
+
+        # Check OnosTopology
+        topoResult = checkTopology( main, expectedLink )
+
+        # Ping hosts
+        pingResult = pingResult and pingallHosts( main, hostNames )
+
+        intentResult = checkIntentState( main, intentsId )
+
+        # Checks ONOS state in link down
+        if linkDownResult and topoResult and pingResult and intentResult:
+            main.log.info( itemName + ": Successfully brought link down" )
+        else:
+            main.log.error( itemName + ": Failed to bring link down" )
+
+        # link up
+        linkUpResult = link( main, sw1, sw2, "up" )
+        time.sleep( 5 )
+
+        # Verify flows
+        checkFlowsState( main )
+
+        # Check OnosTopology
+        topoResult = checkTopology( main, main.numLinks )
+
+        # Ping hosts
+        pingResult = pingResult and pingallHosts( main, hostNames )
+
+        intentResult = checkIntentState( main, intentsId )
+
+        # Checks ONOS state in link up
+        if linkUpResult and topoResult and pingResult and intentResult:
+            main.log.info( itemName + ": Successfully brought link back up" )
+        else:
+            main.log.error( itemName + ": Failed to bring link back up" )
+
+    # Remove all intents
+    removeIntentResult = removeAllIntents( main, intentsId )
+
+    stepResult = pingResult and linkDownResult and linkUpResult \
+                 and intentResult and removeIntentResult
+
+    return stepResult
+
+def multiToSingleIntent( main,
+                         name,
+                         hostNames,
+                         devices="",
+                         ports=None,
+                         ethType="",
+                         macs=None,
+                         bandwidth="",
+                         lambdaAlloc=False,
+                         ipProto="",
+                         ipAddresses="",
+                         tcp="",
+                         sw1="",
+                         sw2="",
+                         expectedLink=0 ):
+    """
+        Verify Single to Multi Point intents
+        NOTE:If main.hostsData is not defined, variables data should be passed in the
+        same order index wise. All devices in the list should have the same
+        format, either all the devices have its port or it doesn't.
+        eg. hostName = [ 'h1', 'h2' ,..  ]
+            devices = [ 'of:0000000000000001', 'of:0000000000000002', ...]
+            ports = [ '1', '1', ..]
+            ...
+        Description:
+            Verify add-multi-to-single-intent
+        Steps:
+            - Get device ids | ports
+            - Add single to multi point intents
+            - Check intents
+            - Verify flows
+            - Ping hosts
+            - Reroute
+                - Link down
+                - Verify flows
+                - Check topology
+                - Ping hosts
+                - Link up
+                - Verify flows
+                - Check topology
+                - Ping hosts
+            - Remove intents
+        Required:
+            name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+            hostNames - List of host names
+        Optional:
+            devices - List of device ids in the same order as the hosts
+                      in hostNames
+            ports - List of port numbers in the same order as the device in
+                    devices
+            ethType - Ethernet type eg. IPV4, IPV6
+            macs - List of hosts mac address in the same order as the hosts in
+                   hostNames
+            bandwidth - Bandwidth capacity
+            lambdaAlloc - Allocate lambda, defaults to False
+            ipProto - IP protocol
+            ipAddresses - IP addresses of host in the same order as the hosts in
+                          hostNames
+            tcp - TCP ports in the same order as the hosts in hostNames
+            sw1 - First switch to bring down & up for rerouting purpose
+            sw2 - Second switch to bring down & up for rerouting purpose
+            expectedLink - Expected link when the switches are down, it should
+                           be two links lower than the links before the two
+                           switches are down
+    """
+
+    import time
+    import copy
+    assert main, "There is no main variable"
+    assert hostNames, "You must specify hosts"
+    assert devices or main.hostsData, "You must specify devices"
+
+    global itemName
+    itemName = name
+    tempHostsData = {}
+    intentsId = []
+
+    macsDict = {}
+    ipDict = {}
+    if hostNames and devices:
+        if len( hostNames ) != len( devices ):
+            main.log.debug( "hosts and devices does not have the same length" )
+            #print "len hostNames = ", len( hostNames )
+            #print "len devices = ", len( devices )
+            return main.FALSE
+        if ports:
+            if len( ports ) != len( devices ):
+                main.log.error( "Ports and devices does " +
+                                "not have the same length" )
+                #print "len devices = ", len( devices )
+                #print "len ports = ", len( ports )
+                return main.FALSE
+        for i in range( len( devices ) ):
+            macsDict[ devices[ i ] ] = macs[ i ]
+        else:
+            main.log.info( "Device Ports are not specified" )
+    elif hostNames and not devices and main.hostsData:
+        devices = []
+        main.log.info( "singleToMultiIntent function is using main.hostsData" ) 
+        for host in hostNames:
+               devices.append( main.hostsData.get( host ).get( 'location' ) )
+               macsDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+                           main.hostsData.get( host ).get( 'mac' )
+               ipDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+                           main.hostsData.get( host ).get( 'ipAddresses' )
+        #print main.hostsData
+
+    #print 'host names = ', hostNames
+    #print 'devices = ', devices
+    #print "macsDict = ", macsDict
+
+    pingResult = main.TRUE
+    intentResult = main.TRUE
+    removeIntentResult = main.TRUE
+    flowResult = main.TRUE
+    topoResult = main.TRUE
+    linkDownResult = main.TRUE
+    linkUpResult = main.TRUE
+
+    devicesCopy = copy.copy( devices )
+    if ports:
+        portsCopy = copy.copy( ports )
+    main.log.info( itemName + ": Adding single point to multi point intents" )
+    # Adding bidirectional point  intents
+    for i in range( len( devices ) ):
+        egressDevice = devicesCopy[ i ]
+        ingressDeviceList = copy.copy( devicesCopy )
+        ingressDeviceList.remove( egressDevice )
+        if ports:
+            portEgress = portsCopy[ i ]
+            portIngressList = copy.copy( portsCopy )
+            del portIngressList[ i ]
+        else:
+            portEgress = ""
+            portIngressList = None
+        if not macsDict:
+            dstMac = ""
+        else:
+            dstMac = macsDict[ egressDevice ]
+            if dstMac == None:
+                main.log.debug( "There is no MAC in device - " + egressDevice )
+                dstMac = ""
+
+        intentsId.append( main.CLIs[ 0 ].addMultipointToSinglepointIntent(
+                                            ingressDeviceList=ingressDeviceList,
+                                            egressDevice=egressDevice,
+                                            portIngressList=portIngressList,
+                                            portEgress=portEgress,
+                                            ethType=ethType,
+                                            ethDst=dstMac,
+                                            bandwidth=bandwidth,
+                                            lambdaAlloc=lambdaAlloc,
+                                            ipProto=ipProto,
+                                            ipSrc="",
+                                            ipDst="",
+                                            tcpSrc="",
+                                            tcpDst="" ) )
+
+    pingResult = pingallHosts( main, hostNames )
+
+    # Check intents state
+    time.sleep( 30 )
+    intentResult = checkIntentState( main, intentsId )
+
+    # Check intents state again if first check fails...
+    if not intentResult:
+        intentResult = checkIntentState( main, intentsId )
+
+    # Verify flows
+    checkFlowsState( main )
+
+    # Ping hosts
+    pingResult = pingResult and pingallHosts( main, hostNames )
+    # Ping hosts again...
+    pingResult = pingResult and pingallHosts( main, hostNames )
+    time.sleep( 5 )
+
+    # Test rerouting if these variables exist
+    if sw1 and sw2 and expectedLink:
+        # link down
+        linkDownResult = link( main, sw1, sw2, "down" )
+        intentResult = intentResult and checkIntentState( main, intentsId )
+
+        # Verify flows
+        checkFlowsState( main )
+
+        # Check OnosTopology
+        topoResult = checkTopology( main, expectedLink )
+
+        # Ping hosts
+        pingResult = pingResult and pingallHosts( main, hostNames )
+
+        intentResult = checkIntentState( main, intentsId )
+
+        # Checks ONOS state in link down
+        if linkDownResult and topoResult and pingResult and intentResult:
+            main.log.info( itemName + ": Successfully brought link down" )
+        else:
+            main.log.error( itemName + ": Failed to bring link down" )
+
+        # link up
+        linkUpResult = link( main, sw1, sw2, "up" )
+        time.sleep( 5 )
+
+        # Verify flows
+        checkFlowsState( main )
+
+        # Check OnosTopology
+        topoResult = checkTopology( main, main.numLinks )
+
+        # Ping hosts
+        pingResult = pingResult and pingallHosts( main, hostNames )
+
+        intentResult = checkIntentState( main, intentsId )
+
+        # Checks ONOS state in link up
+        if linkUpResult and topoResult and pingResult and intentResult:
+            main.log.info( itemName + ": Successfully brought link back up" )
+        else:
+            main.log.error( itemName + ": Failed to bring link back up" )
+
+    # Remove all intents
+    removeIntentResult = removeAllIntents( main, intentsId )
+
+    stepResult = pingResult and linkDownResult and linkUpResult \
+                 and intentResult and removeIntentResult
+
+    return stepResult
+
+def pingallHosts( main, hostList, pingType="ipv4" ):
+    # Ping all host in the hosts list variable
+    print "Pinging : ", hostList
+    pingResult = main.TRUE
+    pingResult = main.Mininet1.pingallHosts( hostList, pingType )
+    return pingResult
+
+def getHostsData( main ):
+    """
+        Use fwd app and pingall to discover all the hosts
+    """
+    import json
+    activateResult = main.TRUE
+    appCheck = main.TRUE
+    getDataResult = main.TRUE
+    main.log.info( "Activating reactive forwarding app " )
+    activateResult = main.CLIs[ 0 ].activateApp( "org.onosproject.fwd" )
+
+    for i in range( main.numCtrls ):
+        appCheck = appCheck and main.CLIs[ i ].appToIDCheck()
+        if appCheck != main.TRUE:
+            main.log.warn( main.CLIs[ i ].apps() )
+            main.log.warn( main.CLIs[ i ].appIDs() )
+
+    pingResult = main.Mininet1.pingall()
+    hostsJson = json.loads( main.CLIs[ 0 ].hosts() )
+    hosts = main.Mininet1.getHosts()
+    for host in hosts:
+        main.hostsData[ host ] = {}
+        main.hostsData[ host ][ 'mac' ] =  \
+            main.Mininet1.getMacAddress( host ).upper()
+        for hostj in hostsJson:
+            if main.hostsData[ host ][ 'mac' ] == hostj[ 'mac' ]:
+                main.hostsData[ host ][ 'id' ] = hostj[ 'id' ]
+                main.hostsData[ host ][ 'vlan' ] = hostj[ 'vlan' ]
+                main.hostsData[ host ][ 'location' ] = \
+                            hostj[ 'location' ][ 'elementId' ] + '/' + \
+                            hostj[ 'location' ][ 'port' ]
+                main.hostsData[ host ][ 'ipAddresses' ] = hostj[ 'ipAddresses' ]
+
+    main.log.info( "Deactivating reactive forwarding app " )
+    deactivateResult = main.CLIs[ 0 ].deactivateApp( "org.onosproject.fwd" )
+    if activateResult and deactivateResult and main.hostsData:
+        main.log.info( "Successfully used fwd app to discover hosts " )
+        getDataResult = main.TRUE
+    else:
+        main.log.info( "Failed to use fwd app to discover hosts " )
+        getDataResult = main.FALSE
+
+    print main.hostsData
+
+    return getDataResult
+
+def checkTopology( main, expectedLink ):
+    statusResult = main.TRUE
+    # Check onos topology
+    main.log.info( itemName + ": Checking ONOS topology " )
+
+    for i in range( main.numCtrls ):
+        topologyResult = main.CLIs[ i ].topology()
+        statusResult = main.ONOSbench.checkStatus( topologyResult,
+                                                   main.numSwitch,
+                                                   expectedLink )\
+                       and statusResult
+    if not statusResult:
+        main.log.error( itemName + ": Topology mismatch" )
+    else:
+        main.log.info( itemName + ": Topology match" )
+    return statusResult
+
+def checkIntentState( main, intentsId ):
+
+    intentResult = main.TRUE
+
+    main.log.info( itemName + ": Checking intents state" )
+    for i in range( main.numCtrls ):
+        intentResult = intentResult and \
+                main.CLIs[ i ].checkIntentState( intentsId=intentsId )
+
+    return intentResult
+
+def checkFlowsState( main ):
+
+    main.log.info( itemName + ": Check flows state" )
+    checkFlowsResult = main.CLIs[ 0 ].checkFlowsState()
+    return checkFlowsResult
+
+def link( main, sw1, sw2, option):
+
+    # link down
+    main.log.info( itemName + ": Bring link " + option + "between " +
+                       sw1 + " and " + sw2 )
+    linkResult = main.Mininet1.link( end1=sw1, end2=sw2, option=option )
+    return linkResult
+
+def removeAllIntents( main, intentsId ):
+    """
+        Remove all intents in the intentsId
+    """
+    import time
+    intentsRemaining = []
+    removeIntentResult = main.TRUE
+    # Remove intents
+    for intent in intentsId:
+        main.CLIs[ 0 ].removeIntent( intentId=intent, purge=True )
+
+    time.sleep( 5 )
+    # Checks if there is remaining intents using intents()
+    intentsRemaining = main.CLIs[ 0 ].intents( jsonFormat=False )
+    # If there is remianing intents then remove intents should fail
+
+    if intentsRemaining:
+        main.log.info( itemName + ": There are " +
+                       str( len( intentsRemaining ) ) + " intents remaining, "
+                       + "failed to remove all the intents " )
+        removeIntentResult = main.FALSE
+        main.log.info( intentsRemaining )
+    else:
+        main.log.info( itemName + ": There are no intents remaining, " +
+                       "successfully removed all the intents." )
+        removeIntentResult = main.TRUE
+    return removeIntentResult
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/FuncIntent/Dependency/__init__.py
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/FuncIntent/Dependency/__init__.py
diff --git a/TestON/tests/FuncIntent/FuncIntent.py b/TestON/tests/FuncIntent/FuncIntent.py
new file mode 100644
index 0000000..d5dc2a7
--- /dev/null
+++ b/TestON/tests/FuncIntent/FuncIntent.py
@@ -0,0 +1,580 @@
+
+# Testing the basic functionality of ONOS Next
+# For sanity and driver functionality excercises only.
+
+import time
+import json
+
+class FuncIntent:
+
+    def __init__( self ):
+        self.default = ''
+
+    def CASE10( self, main ):
+        import time
+        import os
+        import imp
+        """
+        Startup sequence:
+        cell <name>
+        onos-verify-cell
+        onos-remove-raft-log
+        git pull
+        mvn clean install
+        onos-package
+        onos-install -f
+        onos-wait-for-start
+        """
+        global init
+        global globalONOSip
+        try:
+            if type(init) is not bool:
+                init = False
+        except NameError:
+            init = False
+
+        main.wrapper = imp.load_source( 'FuncIntentFunction',
+                                    '/home/admin/ONLabTest/TestON/tests/' +
+                                    'FuncIntent/Dependency/' +
+                                    'FuncIntentFunction.py' )
+        #Local variables
+        cellName = main.params[ 'ENV' ][ 'cellName' ]
+        apps = main.params[ 'ENV' ][ 'cellApps' ]
+        gitBranch = main.params[ 'GIT' ][ 'branch' ]
+        benchIp = os.environ[ 'OCN' ]
+        benchUser = main.params[ 'BENCH' ][ 'user' ]
+        topology = main.params[ 'MININET' ][ 'topo' ]
+        main.numSwitch = int( main.params[ 'MININET' ][ 'switch' ] )
+        main.numLinks = int( main.params[ 'MININET' ][ 'links' ] )
+        main.numCtrls = main.params[ 'CTRL' ][ 'num' ]
+        main.ONOSport = []
+        main.hostsData = {}
+        PULLCODE = False
+        if main.params[ 'GIT' ][ 'pull' ] == 'True':
+            PULLCODE = True
+        main.case( "Setting up test environment" )
+        main.CLIs = []
+        for i in range( 1, int( main.numCtrls ) + 1 ):
+            main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+            main.ONOSport.append( main.params[ 'CTRL' ][ 'port' + str( i ) ] )
+
+        # -- INIT SECTION, ONLY RUNS ONCE -- #
+        if init == False:
+            init = True
+
+            main.scale = ( main.params[ 'SCALE' ] ).split( "," )
+            main.numCtrls = int( main.scale[ 0 ] )
+
+            if PULLCODE:
+                main.step( "Git checkout and pull " + gitBranch )
+                main.ONOSbench.gitCheckout( gitBranch )
+                gitPullResult = main.ONOSbench.gitPull()
+                if gitPullResult == main.ERROR:
+                    main.log.error( "Error pulling git branch" )
+                main.step( "Using mvn clean & install" )
+                cleanInstallResult = main.ONOSbench.cleanInstall()
+                stepResult = cleanInstallResult
+                utilities.assert_equals( expect=main.TRUE,
+                                         actual=stepResult,
+                                         onpass="Successfully compiled " +
+                                                "latest ONOS",
+                                         onfail="Failed to compile " +
+                                                "latest ONOS" )
+            else:
+                main.log.warn( "Did not pull new code so skipping mvn " +
+                               "clean install" )
+
+            globalONOSip = main.ONOSbench.getOnosIps()
+
+        maxNodes = ( len(globalONOSip) - 2 )
+
+        main.numCtrls = int( main.scale[ 0 ] )
+        main.scale.remove( main.scale[ 0 ] )
+
+        main.ONOSip = []
+        for i in range( maxNodes ):
+            main.ONOSip.append( globalONOSip[i] )
+
+        #kill off all onos processes
+        main.log.info( "Safety check, killing all ONOS processes" +
+                       " before initiating enviornment setup" )
+        for i in range(maxNodes):
+            main.ONOSbench.onosDie( globalONOSip[ i ] )
+
+        print "NODE COUNT = ", main.numCtrls
+        main.log.info( "Creating cell file" )
+        cellIp = []
+        for i in range( main.numCtrls ):
+            cellIp.append( str( main.ONOSip[ i ] ) )
+        print cellIp
+        main.ONOSbench.createCellFile( benchIp, cellName, "",
+                                       str( apps ), *cellIp )
+
+        main.step( "Apply cell to environment" )
+        cellResult = main.ONOSbench.setCell( cellName )
+        verifyResult = main.ONOSbench.verifyCell()
+        stepResult = cellResult and verifyResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully applied cell to " + \
+                                        "environment",
+                                 onfail="Failed to apply cell to environment " )
+
+        main.step( "Creating ONOS package" )
+        packageResult = main.ONOSbench.onosPackage()
+        stepResult = packageResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully created ONOS package",
+                                 onfail="Failed to create ONOS package" )
+
+        main.step( "Uninstalling ONOS package" )
+        onosUninstallResult = main.TRUE
+        for i in range( main.numCtrls ):
+            onosUninstallResult = onosUninstallResult and \
+                    main.ONOSbench.onosUninstall( nodeIp=main.ONOSip[ i ] )
+        stepResult = onosUninstallResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully uninstalled ONOS package",
+                                 onfail="Failed to uninstall ONOS package" )
+        time.sleep( 5 )
+        main.step( "Installing ONOS package" )
+        onosInstallResult = main.TRUE
+        for i in range( main.numCtrls ):
+            onosInstallResult = onosInstallResult and \
+                    main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+        stepResult = onosInstallResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully installed ONOS package",
+                                 onfail="Failed to install ONOS package" )
+
+        time.sleep( 20 )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
+        main.step( "Start ONOS cli" )
+        cliResult = main.TRUE
+        for i in range( main.numCtrls ):
+            cliResult = cliResult and \
+                        main.CLIs[i].startOnosCli( main.ONOSip[ i ] )
+        stepResult = cliResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully start ONOS cli",
+                                 onfail="Failed to start ONOS cli" )
+
+    def CASE9( self, main ):
+        '''
+            Report errors/warnings/exceptions
+        '''
+        main.log.info( "Error report: \n" )
+        main.ONOSbench.logReport( globalONOSip[0],
+                [ "INFO", "FOLLOWER", "WARN", "flow", "ERROR" , "Except" ],
+                "s" )
+        #main.ONOSbench.logReport( globalONOSip[1], [ "INFO" ], "d" )
+
+    def CASE11( self, main ):
+        """
+            Start mininet
+        """
+        main.log.report( "Start Mininet topology" )
+        main.log.case( "Start Mininet topology" )
+
+        main.step( "Starting Mininet Topology" )
+        topoResult = main.Mininet1.startNet( topoFile=topology )
+        stepResult = topoResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully loaded topology",
+                                 onfail="Failed to load topology" )
+        # Exit if topology did not load properly
+        if not topoResult:
+            main.cleanup()
+            main.exit()
+
+    def CASE12( self, main ):
+        """
+            Assign mastership to controllers
+        """
+        import re
+
+        main.case( "Assign switches to controllers" )
+        main.step( "Assigning switches to controllers" )
+        assignResult = main.TRUE
+        for i in range( 1, ( main.numSwitch + 1 ) ):
+            main.Mininet1.assignSwController( sw=str( i ),
+                                              count=1,
+                                              ip1=main.ONOSip[ 0 ],
+                                              port1=main.ONOSport[ 0 ] )
+        for i in range( 1, ( main.numSwitch + 1 ) ):
+            response = main.Mininet1.getSwController( "s" + str( i ) )
+            print( "Response is " + str( response ) )
+            if re.search( "tcp:" + main.ONOSip[ 0 ], response ):
+                assignResult = assignResult and main.TRUE
+            else:
+                assignResult = main.FALSE
+        stepResult = assignResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully assigned switches" +
+                                        "to controller",
+                                 onfail="Failed to assign switches to " +
+                                        "controller" )
+    def CASE13( self, main ):
+        """
+            Discover all hosts and store its data to a dictionary
+        """
+        main.case( "Discover all hosts" )
+
+        stepResult = main.TRUE
+        main.step( "Discover all hosts using pingall " )
+        stepResult = main.wrapper.getHostsData( main )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully discovered hosts",
+                                 onfail="Failed to discover hosts" )
+
+    def CASE1001( self, main ):
+        """
+            Add host intents between 2 host:
+                - Discover hosts
+                - Add host intents
+                - Check intents
+                - Verify flows
+                - Ping hosts
+                - Reroute
+                    - Link down
+                    - Verify flows
+                    - Check topology
+                    - Ping hosts
+                    - Link up
+                    - Verify flows
+                    - Check topology
+                    - Ping hosts
+                - Remove intents
+        """
+        import time
+        import json
+        import re
+
+        # Assert variables - These variable's name|format must be followed
+        # if you want to use the wrapper function
+        assert main, "There is no main"
+        assert main.CLIs, "There is no main.CLIs"
+        assert main.Mininet1, "Mininet handle should be named Mininet1"
+        assert main.numSwitch, "Placed the total number of switch topology in \
+                                main.numSwitch"
+
+        main.case( "Add host intents between 2 host" )
+
+        stepResult = main.TRUE
+        main.step( "IPV4: Add host intents between h1 and h9" )
+        stepResult = main.wrapper.hostIntent( main,
+                                              name='IPV4',
+                                              host1='h1',
+                                              host2='h9',
+                                              host1Id='00:00:00:00:00:01/-1',
+                                              host2Id='00:00:00:00:00:09/-1',
+                                              sw1='s5',
+                                              sw2='s2',
+                                              expectedLink=18 )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="IPV4: Add host intent successful",
+                                 onfail="IPV4: Add host intent failed" )
+        stepResult = main.TRUE
+
+        main.step( "DUALSTACK1: Add host intents between h3 and h11" )
+        stepResult = main.wrapper.hostIntent( main,
+                                              name='DUALSTACK',
+                                              host1='h3',
+                                              host2='h11',
+                                              host1Id='00:00:00:00:00:03/-1',
+                                              host2Id='00:00:00:00:00:0B/-1',
+                                              sw1='s5',
+                                              sw2='s2',
+                                              expectedLink=18 )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="DUALSTACK1: Add host intent" +
+                                        " successful",
+                                 onfail="DUALSTACK1: Add host intent failed" )
+
+        stepResult = main.TRUE
+        main.step( "DUALSTACK2: Add host intents between h1 and h9" )
+        stepResult = main.wrapper.hostIntent( main,
+                                              name='DUALSTACK2',
+                                              host1='h1',
+                                              host2='h11',
+                                              sw1='s5',
+                                              sw2='s2',
+                                              expectedLink=18 )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="DUALSTACK2: Add host intent" +
+                                        " successful",
+                                 onfail="DUALSTACK2: Add host intent failed" )
+
+    def CASE1002( self, main ):
+        """
+            Add point intents between 2 hosts:
+                - Get device ids | ports
+                - Add point intents
+                - Check intents
+                - Verify flows
+                - Ping hosts
+                - Reroute
+                    - Link down
+                    - Verify flows
+                    - Check topology
+                    - Ping hosts
+                    - Link up
+                    - Verify flows
+                    - Check topology
+                    - Ping hosts
+                - Remove intents
+        """
+        import time
+        import json
+        import re
+
+        # Assert variables - These variable's name|format must be followed
+        # if you want to use the wrapper function
+        assert main, "There is no main"
+        assert main.CLIs, "There is no main.CLIs"
+        assert main.Mininet1, "Mininet handle should be named Mininet1"
+        assert main.numSwitch, "Placed the total number of switch topology in \
+                                main.numSwitch"
+
+        main.case( "Add point intents between 2 devices" )
+
+        stepResult = main.TRUE
+        main.step( "IPV4: Add point intents between h1 and h9" )
+        stepResult = main.wrapper.pointIntent(
+                                       main,
+                                       name="IPV4",
+                                       host1="h1",
+                                       host2="h9",
+                                       deviceId1="of:0000000000000005/1",
+                                       deviceId2="of:0000000000000006/1",
+                                       port1="",
+                                       port2="",
+                                       ethType="IPV4",
+                                       mac1="00:00:00:00:00:01",
+                                       mac2="00:00:00:00:00:09",
+                                       bandwidth="",
+                                       lambdaAlloc=False,
+                                       ipProto="",
+                                       ip1="",
+                                       ip2="",
+                                       tcp1="",
+                                       tcp2="",
+                                       sw1="s5",
+                                       sw2="s2",
+                                       expectedLink=18 )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="IPV4: Add point intent successful",
+                                 onfail="IPV4: Add point intent failed" )
+
+        stepResult = main.TRUE
+        main.step( "DUALSTACK1: Add point intents between h1 and h9" )
+        stepResult = main.wrapper.pointIntent(
+                                       main,
+                                       name="DUALSTACK1",
+                                       host1="h3",
+                                       host2="h11",
+                                       deviceId1="of:0000000000000005",
+                                       deviceId2="of:0000000000000006",
+                                       port1="3",
+                                       port2="3",
+                                       ethType="IPV4",
+                                       mac1="00:00:00:00:00:03",
+                                       mac2="00:00:00:00:00:0B",
+                                       bandwidth="",
+                                       lambdaAlloc=False,
+                                       ipProto="",
+                                       ip1="",
+                                       ip2="",
+                                       tcp1="",
+                                       tcp2="",
+                                       sw1="s5",
+                                       sw2="s2",
+                                       expectedLink=18 )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="DUALSTACK1: Add point intent" +
+                                        " successful",
+                                 onfail="DUALSTACK1: Add point intent failed" )
+
+    def CASE1003( self, main ):
+        """
+            Add single point to multi point intents
+                - Get device ids
+                - Add single point to multi point intents
+                - Check intents
+                - Verify flows
+                - Ping hosts
+                - Reroute
+                    - Link down
+                    - Verify flows
+                    - Check topology
+                    - Ping hosts
+                    - Link up
+                    - Verify flows
+                    - Check topology
+                    - Ping hosts
+                - Remove intents
+        """
+        assert main, "There is no main"
+        assert main.CLIs, "There is no main.CLIs"
+        assert main.Mininet1, "Mininet handle should be named Mininet1"
+        assert main.numSwitch, "Placed the total number of switch topology in \
+                                main.numSwitch"
+
+        main.case( "Add single point to multi point intents between devices" )
+
+        stepResult = main.TRUE
+        main.step( "IPV4: Add single point to multi point intents" )
+        hostNames = [ 'h8', 'h16', 'h24' ]
+        devices = [ 'of:0000000000000005/8', 'of:0000000000000006/8', \
+                    'of:0000000000000007/8' ]
+        macs = [ '00:00:00:00:00:08', '00:00:00:00:00:10', '00:00:00:00:00:18' ]
+        stepResult = main.wrapper.singleToMultiIntent(
+                                         main,
+                                         name="IPV4",
+                                         hostNames=hostNames,
+                                         devices=devices,
+                                         ports=None,
+                                         ethType="IPV4",
+                                         macs=macs,
+                                         bandwidth="",
+                                         lambdaAlloc=False,
+                                         ipProto="",
+                                         ipAddresses="",
+                                         tcp="",
+                                         sw1="s5",
+                                         sw2="s2",
+                                         expectedLink=18 )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="IPV4: Successfully added single point"
+                                        + " to multi point intents",
+                                 onfail="IPV4: Failed to add single point" +
+                                        " to multi point intents" )
+
+        main.step( "IPV4_2: Add single point to multi point intents" )
+        hostNames = [ 'h8', 'h16', 'h24' ]
+        stepResult = main.wrapper.singleToMultiIntent(
+                                         main,
+                                         name="IPV4",
+                                         hostNames=hostNames,
+                                         ethType="IPV4",
+                                         lambdaAlloc=False )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="IPV4_2: Successfully added single point"
+                                        + " to multi point intents",
+                                 onfail="IPV4_2: Failed to add single point" +
+                                        " to multi point intents" )
+    def CASE1004( self, main ):
+        """
+            Add multi point to single point intents
+                - Get device ids
+                - Add multi point to single point intents
+                - Check intents
+                - Verify flows
+                - Ping hosts
+                - Reroute
+                    - Link down
+                    - Verify flows
+                    - Check topology
+                    - Ping hosts
+                    - Link up
+                    - Verify flows
+                    - Check topology
+                    - Ping hosts
+                - Remove intents
+        """
+        assert main, "There is no main"
+        assert main.CLIs, "There is no main.CLIs"
+        assert main.Mininet1, "Mininet handle should be named Mininet1"
+        assert main.numSwitch, "Placed the total number of switch topology in \
+                                main.numSwitch"
+
+        main.case( "Add multi point to single point intents between devices" )
+
+        stepResult = main.TRUE
+        main.step( "IPV4: Add multi point to single point intents" )
+        hostNames = [ 'h8', 'h16', 'h24' ]
+        devices = [ 'of:0000000000000005/8', 'of:0000000000000006/8', \
+                    'of:0000000000000007/8' ]
+        macs = [ '00:00:00:00:00:08', '00:00:00:00:00:10', '00:00:00:00:00:18' ]
+        stepResult = main.wrapper.multiToSingleIntent(
+                                         main,
+                                         name="IPV4",
+                                         hostNames=hostNames,
+                                         devices=devices,
+                                         ports=None,
+                                         ethType="IPV4",
+                                         macs=macs,
+                                         bandwidth="",
+                                         lambdaAlloc=False,
+                                         ipProto="",
+                                         ipAddresses="",
+                                         tcp="",
+                                         sw1="s5",
+                                         sw2="s2",
+                                         expectedLink=18 )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="IPV4: Successfully added multi point"
+                                        + " to single point intents",
+                                 onfail="IPV4: Failed to add multi point" +
+                                        " to single point intents" )
+
+        main.step( "IPV4_2: Add multi point to single point intents" )
+        hostNames = [ 'h8', 'h16', 'h24' ]
+        stepResult = main.wrapper.multiToSingleIntent(
+                                         main,
+                                         name="IPV4",
+                                         hostNames=hostNames,
+                                         ethType="IPV4",
+                                         lambdaAlloc=False )
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="IPV4_2: Successfully added multi point"
+                                        + " to single point intents",
+                                 onfail="IPV4_2: Failed to add multi point" +
+                                        " to single point intents" )
diff --git a/TestON/tests/FuncIntent/FuncIntent.topo b/TestON/tests/FuncIntent/FuncIntent.topo
new file mode 100755
index 0000000..e6613de
--- /dev/null
+++ b/TestON/tests/FuncIntent/FuncIntent.topo
@@ -0,0 +1,52 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+                <home>~/ONOS</home>
+            </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli1>
+
+        <ONOScli2>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli2>
+
+         <ONOScli3>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli3>
+
+        <Mininet1>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>MininetCliDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </Mininet1>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/FuncIntent/__init__.py
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/FuncIntent/__init__.py
diff --git a/TestON/tests/FuncPlatform/Dependency/App.py b/TestON/tests/FuncPlatform/Dependency/App.py
new file mode 100644
index 0000000..5410c32
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/App.py
@@ -0,0 +1,68 @@
+"""
+Methods related to application interaction
+
+"""
+
+
+def __init__( self ):
+    self.ip = '127.0.0.1'
+
+def activate( apps, nodeToActivateFrom=0 ):
+    """
+    Activate specified applications from node specified
+
+    Ex) apps = ['metrics', 'fwd']
+        nodeToActivateFrom = range( 0, nodes )
+    """
+    if isinstance( apps, ( int, basestring ) ):
+        main.log.error( 'Please pass in a list of strings for args' )
+        return main.FALSE
+
+    if not isinstance( nodeToActivateFrom, ( int ) ) or \
+            nodeToActivateFrom < 0:
+        main.log.error( 'Incorrect node specified' )
+        return main.FALSE
+
+    # TODO: Start log capture and listen for exceptions
+    #       and errors. Also investigate possible keywords
+    #       to listen for when activating applications
+    
+
+    for app in apps:
+        # Check if app str in appList is in the main scope
+        # definition main.appList
+        if app not in main.appList:
+            main.log.error( 'Invalid app name given' )
+            return main.FALSE
+     
+        try:
+            # NOTE: assumes node 1 is always activating application
+            appOutput = main.CLIs[nodeToActivateFrom].activateApp( 
+                main.appList[app] ) 
+        except KeyError:
+            main.log.error( 'There was an error with the key '+
+                    str(app) + '. Check the appList dictionary' )
+            return main.FALSE
+        except Exception:
+            main.log.error( 'Uncaught exception error while ' +
+                    'activating applications: ' + str(app) )
+            return main.FALSE
+
+    return main.TRUE
+
+def deactivate( apps, nodeToDeactivateFrom=0 ):
+    """
+    Deactivate specified applications from node specified
+
+    """
+    main.log.report( 'deactivate implment me' )
+
+def isAppInstallSuccess():
+    """
+    Check the app list across clusters to determine
+    that apps have been installed successfully
+
+    """
+
+    main.log.report( 'isAppInstallSuccess implement me' )
+
diff --git a/TestON/tests/FuncPlatform/Dependency/Logger.py b/TestON/tests/FuncPlatform/Dependency/Logger.py
new file mode 100644
index 0000000..22d3f61
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Logger.py
@@ -0,0 +1,57 @@
+
+def __init__( self ):
+    self.ip = '127.0.0.1'
+
+def checkOnosLog( nodeIp, option='',
+        outputType=0):
+    """
+    Listens to the log for any Errors and Exceptions.
+
+    Runs 'onos-check-logs <option>'
+    This script only returns if there are any errors
+    or exceptions
+
+    outputType
+        0: Return output of log
+        1: Return (#Errors, #Exceptions, #Warn)
+
+    """
+    if not isinstance( option, basestring ):
+        main.log.error( 'Incorrect grep format specified' )
+        return main.FALSE
+
+    try:
+        main.log.info( 'Starting Onos-log listening for '+
+                str(option) )
+        cmd = 'onos-check-logs ' + str(nodeIp) + ' old'
+        if outputType == 0:
+            main.ONOSbench.handle.sendline( cmd )
+            main.ONOSbench.handle.expect( cmd )
+            main.ONOSbench.handle.expect('\$')
+            logResult = main.ONOSbench.handle.before
+            return logResult
+        elif outputType == 1:
+            # Important in assertion criteria
+            # to determine how much warn / error is 
+            # acceptable
+            return 'Implement option 1'
+        else:
+            main.log.error( 'Incorrect outputType specified' )
+            return main.FALSE
+
+    except Exception:
+        main.log.exception( self.name + ': Uncaught exception' ) 
+        main.cleanup()
+        main.exit()
+
+def setLogLevel( level ):
+    """
+    Set the log level of onos
+    """
+    main.log.info( 'setLogLevel implement me' )
+
+def getLogReport( nodeIp, searchTerms ):
+    """
+    Refer to CLI driver for 'logReport'
+    """
+    main.log.info( 'getLogReport - implement me!' )
diff --git a/TestON/tests/FuncPlatform/Dependency/Shutdown.py b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
new file mode 100644
index 0000000..ba77bbd
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
@@ -0,0 +1,19 @@
+
+def __init__( self ):
+    self.ip = '127.0.0.1'
+
+def killOnosNodes( nodeIps ):
+    """
+    Kill all components of Onos on 
+    given list of ips
+    
+    Ex) nodeIps = ['10.0.0.1', '10.0.0.2']
+    """
+    killResult = main.TRUE
+    
+    for node in nodeIps:
+        killResult = killResult and main.ONOSbench.onosDie( node )
+        if killResult == main.TRUE:
+            main.log.info( str(node) + ' was killed' ) 
+
+    return killResult
diff --git a/TestON/tests/FuncPlatform/Dependency/Startup.py b/TestON/tests/FuncPlatform/Dependency/Startup.py
new file mode 100644
index 0000000..f3f90a9
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Startup.py
@@ -0,0 +1,219 @@
+
+"""
+Startup related methods for ONOS
+
+Guidelines:
+    * Group sequential functionalities together
+    * Methods should not be dependent on platform
+    * Return main.TRUE on success or comprehensive error message 
+      on failure (TBD)
+    * All methods should be consistent in expected behavior
+"""
+import time
+import json
+
+def __init__( self ):
+    self.ip = '127.0.0.1' 
+
+def gitPullAndMci( branchName, commitLog=False ):
+    """
+    Pull from branch repository specified and compile changes
+    If commitLog is True, report commit information
+
+    Any errors / warnings will be handled by respective 
+    driver function calls
+    """
+    co = main.ONOSbench.gitCheckout( branchName )
+    gp = main.ONOSbench.gitPull()
+    ci = main.ONOSbench.cleanInstall() 
+
+    if co and gp and ci == main.TRUE:
+        if commitLog:
+            main.log.report( 'Commit information - ' )
+            main.ONOSbench.getVersion(report=True)
+        
+        return main.TRUE
+    
+    else:
+        # TODO: Comprehensive error message
+        return 'git pull and mci failed'
+
+def initOnosStartupSequence( cellName, appStr, benchIp, mnIp, onosIps ):
+    """
+    Startup sequence includes the following:
+        * Create cell file
+        * Set cell variables on ONOS bench
+        * Verify cell
+        * Create ONOS package
+        * Force install ONOS package
+        * Start ONOS service
+        * Start ONOS cli
+    
+    Also verifies that Onos is up and running by 
+    'isup' driver function which executs 
+    'onos-wait-for-start'
+    """
+
+    # NOTE: leave out create cell file until bug addressed
+    #cf = main.ONOSbench.createCellFile( benchIp, cellName, mnIp, 
+    #        str(appStr), *onosIps )
+    numNodes = len(onosIps) 
+
+    sc = main.ONOSbench.setCell( cellName )
+    vc = main.ONOSbench.verifyCell()
+    op = main.ONOSbench.onosPackage()
+    for addr in onosIps:
+        oi = main.ONOSbench.onosInstall( node = addr )
+    
+    time.sleep( 5 )
+   
+    iu = main.TRUE
+    for node in onosIps:
+        iu = iu and main.ONOSbench.isup( node )
+   
+    cli = main.TRUE
+    for node in range( 0, numNodes ):
+        cli = cli and main.CLIs[node].startOnosCli( onosIps[node] )
+
+    # Check if all nodes are discovered correctly using
+    # 'nodes' command in Onos Cli
+    na = main.TRUE
+    try:
+        nodeCmdJson = json.loads( main.CLIs[0].nodes() )
+        for node in nodeCmdJson:
+            if node['state'] != 'ACTIVE':
+                main.log.warn( str( node['id'] ) + 
+                        ' Node is not in ACTIVE state.' )
+                na = main.FALSE
+        if na != main.FALSE:
+            main.log.info( 'All nodes discovered successfully' )
+    except Exception:
+        main.log.error( 'nodes command did not execute properly' )
+        return main.FALSE
+
+    if sc and vc and op and oi and iu and cli and na == main.TRUE:
+        return main.TRUE
+    else:
+        return main.FALSE
+
+def installOnosFromTar( wgetAddr, nodeIps ):
+    """
+    Install Onos directly from tar.gz file.
+    Due to the nature of the specific steps required 
+    to startup Onos in this fashion, all commands
+    required to start Onos from tar.gz will be
+    grouped in this method. 
+
+    1) wget latest onos tar.gz on onos node
+    2) untar package
+    3) specify onos-config cluster
+    4) start onos via onos-service
+    5) form onos cluster using onos-form-cluster
+    6) check for successful startup
+
+    Specify the download link for the tar.gz.
+    Provide a list of nodeIps
+
+    Ex) wgetAddr = 'https://mytargzdownload.com/file.tar.gz'
+        nodeIps = ['10.0.0.1', '10.0.0.2']
+    """
+    if isinstance( nodeIps, ( int, basestring ) ):
+        main.log.error( 'Please pass in a list of string nodes' )
+        return main.FALSE
+
+    # Obtain filename from provided address
+    # assumes that filename is separated by '/' character
+    f_name = ''
+    addr = str( wgetAddr ).split('/')
+    for phrase in addr:
+        if 'tar.gz' in phrase:
+            f_name = str( phrase )
+            main.log.info( 'Using ' + f_name + ' as file' ) 
+
+    clusterCount = len( nodeIps )
+
+    main.log.info( 'Initiating Onos installation sequence ' +
+            'using tar.gz ... This may take a few minutes' )
+
+    for node in range( 0, clusterCount ):
+        # Use the wgetAddr to download a new tar.gz from server
+        try:
+            main.ONOSnode[node].handle.sendline( 'wget ' + wgetAddr )
+            main.ONOSnode[node].handle.expect( 'saved' )
+            main.ONOSnode[node].handle.expect( '\$' )
+            main.log.info( 'Successfully downloaded tar.gz ' +
+                    'on node: ' + str( main.ONOSips[node] ) ) 
+        except Exception:
+            # NOTE: Additional exception may be appropriate 
+            main.log.error( 'Uncaught exception while ' +
+                    'downloading Onos tar.gz: ' + 
+                    main.ONOSnode[node].handle.before )
+            return main.FALSE
+
+    for node in range( 0, clusterCount ):
+        # Untar files on all nodes, then enter the 
+        # newly created directory
+        try:
+            main.ONOSnode[node].handle.sendline( 'tar zxvf ' + f_name )
+            # Verbose output of tar will contain some onos returns
+            main.ONOSnode[node].handle.expect( 'onos' )
+            main.ONOSnode[node].handle.expect( '\$' )
+            # NOTE: Making an assumption here: 
+            #       the directory created by tar file has a name that
+            #       starts with 'onos-' followed by version number
+            #       '1.2.0'. If this is NOT true, this method must
+            #       be changed to enter the correct directory.
+            #       The directory name is currently dynamic 
+            #       and depends on the day at which you downloaded
+            #       the tar file
+            # Enter onos- wildcard to disregard its version / date 
+            #       suffix
+            main.ONOSnode[node].handle.sendline( 'cd onos-*' )
+            main.ONOSnode[node].handle.expect( '\$' )
+            
+            main.ONOSnode[node].handle.sendline( 'pwd' )
+            main.ONOSnode[node].handle.expect( 'pwd' )
+            main.ONOSnode[node].handle.expect( '\$' )
+            pwd = main.ONOSnode[node].handle.before 
+            if 'onos' in str(pwd):
+                main.log.info( 'tar zxvf ' + f_name + ' successful ' +
+                        'on node ' + str( main.ONOSips[node] ) )
+
+        except Exception:
+            main.log.error( 'Uncaught exception while executing ' +
+                    'tar zxvf of ' + f_name + ': ' +
+                    main.ONOSnode[node].handle.before +
+                    main.ONOSnode[node].handle.after)
+            return main.FALSE
+
+    for node in range( 0, clusterCount ):
+        try:
+            main.ONOSnode[node].handle.sendline( 'bin/onos-service '+
+                    'server &' )
+            # Send some extra characters to run the process
+            main.ONOSnode[node].handle.sendline( '' )
+            main.ONOSnode[node].handle.sendline( '' )
+            main.ONOSnode[node].handle.expect( '\$' )
+        except Exception:
+            main.log.error( 'Uncaught exception while executing ' +
+                    'onos-service server command ' + 
+                    str( main.ONOSnode[node].handle.before ) )
+            return main.FALSE
+    
+    iu = main.TRUE
+    for node in nodeIps:
+        iu = iu and main.ONOSbench.isup( node )
+
+    if iu == main.TRUE: 
+        return main.TRUE
+    else:
+        return main.FALSE
+    
+def addAndStartOnosNode( nodeIps ):
+    """
+    A scale-out scenario that adds specified list of 
+    nodes and starts those instances.
+
+    Ex) nodeIps = ['10.0.0.2', '10.0.0.3', 10.0.0.4']
+    """
+    main.log.info( 'addAndStartOnosNode implement me!' )
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.params b/TestON/tests/FuncPlatform/FuncPlatform.params
new file mode 100644
index 0000000..3895660
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.params
@@ -0,0 +1,63 @@
+<PARAMS>
+    <testcases>1,2,3,4</testcases>
+    <DEP>
+        <startupSrc>
+            /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Startup.py
+        </startupSrc>
+        <startupClassName>
+            Startup
+        </startupClassName>
+        
+        <appSrc>
+            /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/App.py
+        </appSrc>
+        <appClassName>
+            App
+        </appClassName>
+    
+        <logSrc>
+            /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Logger.py
+        </logSrc>
+        <logClassName>
+            Log
+        </logClassName>
+   
+        <shutdownSrc>
+            /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Shutdown.py
+        </shutdownSrc>
+        <shutdownClassName>
+            Shutdown
+        </shutdownClassName>
+
+        <targz>
+        http://downloads.onosproject.org/nightly/onos-1.2.0.latest-NIGHTLY.tar.gz
+        </targz>
+    </DEP> 
+
+    <CTRL>
+        <num>3</num>
+        <port>6633</port>
+        <ip1>10.128.174.1</ip1>
+        <ip2>10.128.174.2</ip2>
+        <ip3>10.128.174.3</ip3>
+    </CTRL>
+  
+    <MN>
+        <ip>10.128.10.90</ip>
+    </MN>
+
+    <BENCH>
+        <ip>10.128.174.10</ip>
+    </BENCH>
+
+    <CELL>
+        <name>func_plat</name>
+        <appStr>"drivers,openflow"</appStr>
+    </CELL>
+
+    <GIT>
+        <branchName>master</branchName>
+        <pull>off</pull>
+    </GIT>
+
+</PARAMS>
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.py b/TestON/tests/FuncPlatform/FuncPlatform.py
new file mode 100644
index 0000000..60b1aa0
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.py
@@ -0,0 +1,194 @@
+"""
+FuncPlatform
+
+A functional test designed to test the environment and 
+gather information on startup -> shutdown related issues.
+
+Future works may also include security mode startup /
+shutdown check and cfg get and set.
+
+Abstracting the collection of commands that go hand in hand 
+should allow easy rearrangement of steps to replicate or 
+create scenarios. 
+For example:
+    CASE N - Represents a particular scenario
+        Steps - Represents abstraction methods called from 
+                dependency
+        1. Bring ONOS 1 up
+        2. Activate application X
+        3. Activate application Y
+        4. Deactivate application X
+
+The ideal platform test script should have incredible
+robustness to possible exceptions and report the most
+useful error messages. 
+
+contributers to contact for help:
+andrew@onlab.us
+"""
+
+class FuncPlatform:
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        Main scope initialization case
+        Must include to run any other test cases
+        """   
+        import imp 
+
+        # NOTE: Hardcoded application name subject to change
+        #       closely monitor and make changes when necessary
+        #       (or implement ways to dynamically get names)
+        main.appList = { 
+            'bgprouter' : 'org.onosproject.bgprouter',
+            'config' : 'org.onosproject.config',
+            'cordfabric' : 'org.onosproject.cordfabric',
+            'demo' : 'org.onosproject.demo',
+            'distributedprimitives' : 'org.onosproject.distributedprimitives',
+            'election' : 'org.onosproject.election',
+            'flowrule' : 'org.onosproject.flowrule',
+            'fwd' : 'org.onosproject.fwd',
+            'intentperf' : 'org.onosproject.intentperf',
+            'messagingperf' : 'org.onosproject.messagingperf',
+            'metrics' : 'org.onosproject.metrics',
+            'mobility' : 'org.onosproject.mobility',
+            'netconf' : 'org.onosproject.netconf', 
+            'null' : 'org.onosproject.null',
+            'optical' : 'org.onosproject.optical',
+            'pcep' : 'org.onosproject.pcep',
+            'proxyarp' : 'org.onosproject.proxyarp',
+            'reactive.routing' : 'org.onosproject.reactive.routing',
+            'sdnip' : 'org.onosproject.sdnip',
+            'segmentrouting' : 'org.onosproject.segmentrouting',
+            'tunnel' : 'org.onosproject.tunnel',
+            'virtualbng' : 'org.onosproject.virtualbng',
+            'xosintegration' : 'org.onosproject.xosintegration'
+            } 
+        # List of ONOS ip's specififed in params
+        main.ONOSips = [] 
+        main.CLIs = []
+        main.ONOSnode = []
+
+        for node in range( 0, int(main.params['CTRL']['num']) ):
+            main.ONOSips.append( main.params['CTRL']['ip'+str(node+1)] )
+            main.CLIs.append(
+                    getattr( main, 'ONOS' + str(node+1) + 'cli' ) )
+            main.ONOSnode.append(
+                    getattr( main, 'ONOS' + str(node+1) ) )
+       
+        # Application source and name definitions
+        startupSrc = main.params['DEP']['startupSrc']
+        startupClassName = main.params['DEP']['startupClassName']
+        
+        appClassName = main.params['DEP']['appClassName']
+        appSrc = main.params['DEP']['appSrc']
+
+        logClassName = main.params['DEP']['logClassName']
+        logSrc = main.params['DEP']['logSrc']
+
+        shutdownClassName = main.params['DEP']['shutdownClassName']
+        shutdownSrc = main.params['DEP']['shutdownSrc']
+
+        # Importing dependency class(es)
+        # Refer to source files in Dependency folder to
+        # make changes to its respective methods
+        # Be weary of naming collisions
+        try:
+            main.startup = imp.load_source( startupClassName, startupSrc )
+            main.app = imp.load_source( appClassName, appSrc )
+            main.onosLog = imp.load_source( logClassName, logSrc )
+            main.shutdown = imp.load_source( shutdownClassName, shutdownSrc )
+        except ImportError:
+            main.log.error( 'Error importing class file(s). Please ' +
+                    'check file location' )
+            main.cleanup()
+            main.exit()
+
+    def CASE2( self, main ):
+        import time
+
+        cellName = main.params['CELL']['name']
+        appStr = main.params['CELL']['appStr']
+        benchIp = main.params['BENCH']['ip']
+        branchName = main.params['GIT']['branchName']
+        gitPull = main.params['GIT']['pull']
+        mnIp = main.params['MN']['ip']
+
+        main.case( 'Setup environment and install ONOS' )
+        if gitPull == 'on': 
+            main.step( 'Git pull and clean install' )
+            gitPullResult = main.startup.gitPullAndMci( branchName )
+            utilities.assert_equals( expect=main.TRUE,
+                        actual=gitPullResult,
+                        onpass='Git pull and install successful',
+                        onfail='Git pull and install failed: ' +
+                            str(gitPullResult) )
+        
+        main.step( 'Initiate ONOS startup sequence' )    
+        startupResult = main.startup.initOnosStartupSequence(
+                cellName, appStr, benchIp, mnIp, main.ONOSips )
+        utilities.assert_equals( expect=main.TRUE,
+                        actual=startupResult,
+                        onpass='ONOS startup sequence successful',
+                        onfail='ONOS startup sequence failed: ' +
+                            str(startupResult) )
+        
+    def CASE3( self, main ):
+        import time
+
+        main.case( 'Activate applications and check installation' )
+       
+        # NOTE: Test only
+        # Unceremoniously kill onos 2 
+        main.ONOSbench.onosDie( '10.128.174.2' )
+        
+        time.sleep( 30 )
+
+        main.step( 'Sample Onos log check' )
+        logResult = main.onosLog.checkOnosLog( main.ONOSips[0] )
+        main.log.info( logResult )
+        # TODO: Define pass criteria
+        utilities.assert_equals( expect=main.TRUE,
+                actual=main.TRUE,
+                onpass= 'Logging successful',
+                onfail= 'Logging failed ' )
+
+        # Sample app activation
+        main.step( 'Activating applications metrics and fwd' ) 
+        appList = ['metrics', 'fwd']
+        appResult = main.app.activate( appList ) 
+        utilities.assert_equals( expect=main.TRUE,
+                actual=appResult,
+                onpass= 'App activation of ' + str(appList) + ' successful',
+                onfail= 'App activation failed ' + str(appResult) )
+
+    def CASE4( self, main ):
+        """
+        Download ONOS tar.gz built from latest nightly
+        (following tutorial on wiki) and run ONOS directly on the
+        instance
+        """
+        import imp
+
+        targz = main.params['DEP']['targz']
+        clusterCount = main.params['CTRL']['num']
+       
+        main.case( 'Install ONOS from onos.tar.gz file' )
+
+        main.step( 'Killing all ONOS instances previous started' )
+        killResult = main.shutdown.killOnosNodes( main.ONOSips )
+        utilities.assert_equals( expect=main.TRUE,
+                actual = killResult,
+                onpass = 'All Onos nodes successfully killed',
+                onfail = 'Onos nodes were not successfully killed' )
+
+        main.step( 'Starting ONOS using tar.gz on all nodes' )
+        installResult = main.startup.installOnosFromTar( targz, main.ONOSips )
+        utilities.assert_equals( expect=main.TRUE,
+                actual = installResult,
+                onpass= 'Onos tar.gz installation successful',
+                onfail= 'Onos tar.gz installation failed' )
+
+
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.topo b/TestON/tests/FuncPlatform/FuncPlatform.topo
new file mode 100644
index 0000000..f19e8f8
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.topo
@@ -0,0 +1,82 @@
+<TOPOLOGY>
+    <COMPONENT>
+        
+        <ONOSbench>
+            <host>10.128.174.10</host>
+            <user>admin</user>
+            <password></password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOSbench>
+        
+        <ONOS1>
+            <host>10.128.174.1</host>
+            <user>admin</user>
+            <password></password>
+            <type>OnosDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS1>
+        
+        <ONOS2>
+            <host>10.128.174.2</host>
+            <user>admin</user>
+            <password></password>
+            <type>OnosDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS2>
+
+        <ONOS3>
+            <host>10.128.174.3</host>
+            <user>admin</user>
+            <password></password>
+            <type>OnosDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS3>
+   
+        <ONOS1cli>
+            <host>10.128.174.10</host>
+            <user>admin</user>
+            <password></password>
+            <type>OnosCliDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS1cli>
+        
+        <ONOS2cli>
+            <host>10.128.174.10</host>
+            <user>admin</user>
+            <password></password>
+            <type>OnosCliDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS2cli>
+
+        <ONOS3cli>
+            <host>10.128.174.10</host>
+            <user>admin</user>
+            <password></password>
+            <type>OnosCliDriver</type>
+            <connect_order>8</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS3cli>
+        
+        <MininetOvs>
+            <host>10.128.10.90</host>
+            <user>admin</user>
+            <password></password>
+            <type>MininetCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS>
+                <arg1> --custom topo-perf-2sw.py </arg1>
+                <arg2> --arp --mac --topo mytopo </arg2>
+                <arg3> </arg3>
+                <controller> remote </controller>
+            </COMPONENTS>
+        </MininetOvs>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/FuncStartTemplate/FuncStartTemplate.params b/TestON/tests/FuncStartTemplate/FuncStartTemplate.params
new file mode 100755
index 0000000..310e5f8
--- /dev/null
+++ b/TestON/tests/FuncStartTemplate/FuncStartTemplate.params
@@ -0,0 +1,34 @@
+<PARAMS>
+
+    <testcases>10,9,10,9</testcases>
+
+    <SCALE>3,1</SCALE>
+    <availableNodes>3</availableNodes>
+    <ENV>
+        <cellName>single_func</cellName>
+        <cellApps>drivers,openflow,proxyarp,mobility</cellApps>
+    </ENV>
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+    <CTRL>
+        <num>3</num>
+        <ip1>OC1</ip1>
+        <port1>6633</port1>
+        <ip2>OC2</ip2>
+        <port2>6633</port2>
+        <ip3>OC3</ip3>
+        <port3>6633</port3>
+    </CTRL>
+    <BENCH>
+        <user>admin</user>
+        <ip1>OCN</ip1>
+    </BENCH>
+    <MININET>
+        <switch>7</switch>
+        <links>20</links>
+        <topo>~/mininet/custom/newFuncTopo.py</topo>
+    </MININET>
+
+</PARAMS>
diff --git a/TestON/tests/FuncStartTemplate/FuncStartTemplate.py b/TestON/tests/FuncStartTemplate/FuncStartTemplate.py
new file mode 100644
index 0000000..4beefae
--- /dev/null
+++ b/TestON/tests/FuncStartTemplate/FuncStartTemplate.py
@@ -0,0 +1,231 @@
+
+# Testing the basic functionality of ONOS Next
+# For sanity and driver functionality excercises only.
+
+import time
+import json
+
+time.sleep( 1 )
+
+class FuncStartTemplate:
+
+    def __init__( self ):
+        self.default = ''
+
+    def CASE10( self, main ):
+        import time
+        import os
+        """
+        Startup sequence:
+        cell <name>
+        onos-verify-cell
+        onos-remove-raft-log
+        git pull
+        mvn clean install
+        onos-package
+        onos-install -f
+        onos-wait-for-start
+        """
+        global init
+        global globalONOSip
+        try:
+            if type(init) is not bool:
+                init = False
+        except NameError:
+            init = False
+
+        #Local variables
+        cellName = main.params[ 'ENV' ][ 'cellName' ]
+        apps = main.params[ 'ENV' ][ 'cellApps' ]
+        gitBranch = main.params[ 'GIT' ][ 'branch' ]
+        benchIp = os.environ[ 'OCN' ]
+        benchUser = main.params[ 'BENCH' ][ 'user' ]
+        topology = main.params[ 'MININET' ][ 'topo' ]
+        main.numSwitch = int( main.params[ 'MININET' ][ 'switch' ] )
+        main.numLinks = int( main.params[ 'MININET' ][ 'links' ] )
+        main.numCtrls = main.params[ 'CTRL' ][ 'num' ]
+        PULLCODE = False
+        if main.params[ 'GIT' ][ 'pull' ] == 'True':
+            PULLCODE = True
+        main.case( "Setting up test environment" )
+        main.CLIs = []
+        for i in range( 1, int( main.numCtrls ) + 1 ):
+            main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+
+        # -- INIT SECTION, ONLY RUNS ONCE -- #
+        if init == False:
+            init = True
+
+            main.ONOSport = []
+            main.scale = ( main.params[ 'SCALE' ] ).split( "," )
+            main.numCtrls = int( main.scale[ 0 ] )
+
+            if PULLCODE:
+                main.step( "Git checkout and pull " + gitBranch )
+                main.ONOSbench.gitCheckout( gitBranch )
+                gitPullResult = main.ONOSbench.gitPull()
+                if gitPullResult == main.ERROR:
+                    main.log.error( "Error pulling git branch" )
+                main.step( "Using mvn clean & install" )
+                cleanInstallResult = main.ONOSbench.cleanInstall()
+                stepResult = cleanInstallResult
+                utilities.assert_equals( expect=main.TRUE,
+                                         actual=stepResult,
+                                         onpass="Successfully compiled " +
+                                                "latest ONOS",
+                                         onfail="Failed to compile " +
+                                                "latest ONOS" )
+            else:
+                main.log.warn( "Did not pull new code so skipping mvn " +
+                               "clean install" )
+
+            globalONOSip = main.ONOSbench.getOnosIps()
+        
+        maxNodes = ( len(globalONOSip) - 2 )
+
+        main.numCtrls = int( main.scale[ 0 ] )
+        main.scale.remove( main.scale[ 0 ] )
+
+        main.ONOSip = []
+        for i in range( maxNodes ): 
+            main.ONOSip.append( globalONOSip[i] )
+
+
+        
+        #kill off all onos processes
+        main.log.info( "Safety check, killing all ONOS processes" +
+                       " before initiating enviornment setup" )
+        for i in range(maxNodes):
+            main.ONOSbench.onosDie( globalONOSip[ i ] )
+        
+        """
+        main.step( "Apply cell to environment" )
+        cellResult = main.ONOSbench.setCell( cellName )
+        verifyResult = main.ONOSbench.verifyCell()
+        stepResult = cellResult and verifyResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully applied cell to " + \
+                                        "environment",
+                                 onfail="Failed to apply cell to environment " )
+        """
+        """main.step( "Removing raft logs" )
+        removeRaftResult = main.ONOSbench.onosRemoveRaftLogs()
+        stepResult = removeRaftResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully removed raft logs",
+                                 onfail="Failed to remove raft logs" )
+        """
+
+        print "NODE COUNT = ", main.numCtrls
+        main.log.info( "Creating cell file" )
+        cellIp = []
+        for i in range( main.numCtrls ):
+            cellIp.append( str( main.ONOSip[ i ] ) )
+        print cellIp
+        main.ONOSbench.createCellFile( benchIp, cellName, "",
+                                       str( apps ), *cellIp )
+
+        main.step( "Apply cell to environment" )
+        cellResult = main.ONOSbench.setCell( cellName )
+        verifyResult = main.ONOSbench.verifyCell()
+        stepResult = cellResult and verifyResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully applied cell to " + \
+                                        "environment",
+                                 onfail="Failed to apply cell to environment " )
+
+        main.step( "Creating ONOS package" )
+        packageResult = main.ONOSbench.onosPackage()
+        stepResult = packageResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully created ONOS package",
+                                 onfail="Failed to create ONOS package" )
+
+        main.step( "Uninstalling ONOS package" )
+        onosUninstallResult = main.TRUE
+        for i in range( main.numCtrls ):
+            onosUninstallResult = onosUninstallResult and \
+                    main.ONOSbench.onosUninstall( nodeIp=main.ONOSip[ i ] )
+        stepResult = onosUninstallResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully uninstalled ONOS package",
+                                 onfail="Failed to uninstall ONOS package" )
+        time.sleep( 5 )
+        main.step( "Installing ONOS package" )
+        onosInstallResult = main.TRUE
+        for i in range( main.numCtrls ):
+            onosInstallResult = onosInstallResult and \
+                    main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+        stepResult = onosInstallResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully installed ONOS package",
+                                 onfail="Failed to install ONOS package" )
+
+        time.sleep( 20 )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+        
+        main.step( "Start ONOS cli" )
+        cliResult = main.TRUE
+        for i in range( main.numCtrls ):
+            cliResult = cliResult and \
+                        main.CLIs[i].startOnosCli( main.ONOSip[ i ] )
+        stepResult = cliResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully start ONOS cli",
+                                 onfail="Failed to start ONOS cli" )
+    
+    def CASE9( self, main ): 
+        '''
+            Report errors/warnings/exceptions 
+        '''
+        main.log.info("Error report: \n") 
+        main.ONOSbench.logReport( globalONOSip[0], [ "INFO","FOLLOWER","WARN","flow","ERROR","Except" ], "s" )  
+        #main.ONOSbench.logReport( globalONOSip[1], [ "INFO" ], "d" )
+
+    def CASE11( self, main ):
+        """
+            Start mininet
+        """
+        main.log.report( "Start Mininet topology" )
+        main.log.case( "Start Mininet topology" )
+
+        main.step( "Starting Mininet Topology" )
+        topoResult = main.Mininet1.startNet( topoFile=topology )
+        stepResult = topoResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully loaded topology",
+                                 onfail="Failed to load topology" )
+        # Exit if topology did not load properly
+        if not topoResult:
+            main.cleanup()
+            main.exit()
+
diff --git a/TestON/tests/FuncStartTemplate/FuncStartTemplate.topo b/TestON/tests/FuncStartTemplate/FuncStartTemplate.topo
new file mode 100755
index 0000000..2321da0
--- /dev/null
+++ b/TestON/tests/FuncStartTemplate/FuncStartTemplate.topo
@@ -0,0 +1,52 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+                <home>~/onos</home>
+            </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli1>
+
+        <ONOScli2>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli2>
+
+         <ONOScli3>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli3>
+
+        <Mininet1>
+            <host>OCN</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>MininetCliDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </Mininet1>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/FuncStartTemplate/__init__.py
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/FuncStartTemplate/__init__.py
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 123fd0f..390facf 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -1,11 +1,12 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,14,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
         <cellName>HA</cellName>
     </ENV>
-    <Git>False</Git>
+    <Git> False </Git>
     <branch> master </branch>
     <num_controllers> 7 </num_controllers>
+    <tcpdump> False </tcpdump>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 1205b11..50bf06f 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -18,6 +18,8 @@
 CASE13: Clean up
 CASE14: start election app on all onos nodes
 CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
 """
 
 
@@ -44,9 +46,12 @@
         start cli sessions
         start tcpdump
         """
-        main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
+        main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
                          "initialization" )
         main.case( "Setting up test environment" )
+        main.caseExplaination = "Setup the test environment including " +\
+                                "installing ONOS, starting Mininet and ONOS" +\
+                                "cli sessions."
         # TODO: save all the timers and output them for plotting
 
         # load some variables from the params file
@@ -90,36 +95,73 @@
         verifyResult = main.ONOSbench.verifyCell()
 
         # FIXME:this is short term fix
-        main.log.report( "Removing raft logs" )
+        main.log.info( "Removing raft logs" )
         main.ONOSbench.onosRemoveRaftLogs()
 
-        main.log.report( "Uninstalling ONOS" )
+        main.log.info( "Uninstalling ONOS" )
         for node in nodes:
             main.ONOSbench.onosUninstall( node.ip_address )
 
+        # Make sure ONOS is DEAD
+        main.log.info( "Killing any ONOS processes" )
+        killResults = main.TRUE
+        for node in nodes:
+            killed = main.ONOSbench.onosKill( node.ip_address )
+            killResults = killResults and killed
+
         cleanInstallResult = main.TRUE
         gitPullResult = main.TRUE
 
         main.step( "Starting Mininet" )
-        main.Mininet1.startNet( )
+        mnResult = main.Mininet1.startNet( )
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet Started",
+                                 onfail="Error starting Mininet" )
 
-        main.step( "Compiling the latest version of ONOS" )
+        main.step( "Git checkout and pull " + gitBranch )
         if PULLCODE:
-            main.step( "Git checkout and pull " + gitBranch )
             main.ONOSbench.gitCheckout( gitBranch )
             gitPullResult = main.ONOSbench.gitPull()
-            if gitPullResult == main.ERROR:
-                main.log.error( "Error pulling git branch" )
+            # values of 1 or 3 are good
+            utilities.assert_lesser( expect=0, actual=gitPullResult,
+                                      onpass="Git pull successful",
+                                      onfail="Git pull failed" )
+        main.ONOSbench.getVersion( report=True )
 
-            main.step( "Using mvn clean & install" )
+        main.step( "Using mvn clean install" )
+        cleanInstallResult = main.TRUE
+        if PULLCODE and gitPullResult == main.TRUE:
             cleanInstallResult = main.ONOSbench.cleanInstall()
         else:
             main.log.warn( "Did not pull new code so skipping mvn " +
                            "clean install" )
-        main.ONOSbench.getVersion( report=True )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=cleanInstallResult,
+                                 onpass="MCI successful",
+                                 onfail="MCI failed" )
+        # GRAPHS
+        # NOTE: important params here:
+        #       job = name of Jenkins job
+        #       Plot Name = Plot-HA, only can be used if multiple plots
+        #       index = The number of the graph under plot name
+        job = "HAClusterRestart"
+        plotName = "Plot-HA"
+        graphs = '<ac:structured-macro ac:name="html">\n'
+        graphs += '<ac:plain-text-body><![CDATA[\n'
+        graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+                  '/plot/' + plotName + '/getPlot?index=0' +\
+                  '&width=500&height=300"' +\
+                  'noborder="0" width="500" height="300" scrolling="yes" ' +\
+                  'seamless="seamless"></iframe>\n'
+        graphs += ']]></ac:plain-text-body>\n'
+        graphs += '</ac:structured-macro>\n'
+        main.log.wiki(graphs)
 
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()
+        utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+                                 onpass="ONOS package successful",
+                                 onfail="ONOS package failed" )
 
         main.step( "Installing ONOS package" )
         onosInstallResult = main.TRUE
@@ -127,6 +169,9 @@
             tmpResult = main.ONOSbench.onosInstall( options="-f",
                                                     node=node.ip_address )
             onosInstallResult = onosInstallResult and tmpResult
+        utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+                                 onpass="ONOS install successful",
+                                 onfail="ONOS install failed" )
 
         main.step( "Checking if ONOS is up yet" )
         for i in range( 2 ):
@@ -134,12 +179,15 @@
             for node in nodes:
                 started = main.ONOSbench.isup( node.ip_address )
                 if not started:
-                    main.log.report( node.name + " didn't start!" )
+                    main.log.error( node.name + " didn't start!" )
                     main.ONOSbench.onosStop( node.ip_address )
                     main.ONOSbench.onosStart( node.ip_address )
                 onosIsupResult = onosIsupResult and started
             if onosIsupResult == main.TRUE:
                 break
+        utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+                                 onpass="ONOS startup successful",
+                                 onfail="ONOS startup failed" )
 
         main.log.step( "Starting ONOS CLI sessions" )
         cliResults = main.TRUE
@@ -154,14 +202,19 @@
         for t in threads:
             t.join()
             cliResults = cliResults and t.result
+        utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                 onpass="ONOS cli startup successful",
+                                 onfail="ONOS cli startup failed" )
 
-        main.step( "Start Packet Capture MN" )
-        main.Mininet2.startTcpdump(
-            str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
-            + "-MN.pcap",
-            intf=main.params[ 'MNtcpdump' ][ 'intf' ],
-            port=main.params[ 'MNtcpdump' ][ 'port' ] )
+        if main.params[ 'tcpdump' ].lower() == "true":
+            main.step( "Start Packet Capture MN" )
+            main.Mininet2.startTcpdump(
+                str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+                + "-MN.pcap",
+                intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+                port=main.params[ 'MNtcpdump' ][ 'port' ] )
 
+        main.step( "App Ids check" )
         appCheck = main.TRUE
         threads = []
         for i in range( numControllers ):
@@ -174,22 +227,15 @@
         for t in threads:
             t.join()
             appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
         if appCheck != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
-        case1Result = ( cleanInstallResult and packageResult and
-                        cellResult and verifyResult and onosInstallResult
-                        and onosIsupResult and cliResults and appCheck)
-
-        utilities.assert_equals( expect=main.TRUE, actual=case1Result,
-                                 onpass="Test startup successful",
-                                 onfail="Test startup NOT successful" )
-
-        if case1Result == main.FALSE:
+        if cliResults == main.FALSE:
+            main.log.error( "Failed to start ONOS, stopping test" )
             main.cleanup()
             main.exit()
 
@@ -198,6 +244,7 @@
         Assign mastership to controllers
         """
         import re
+        import time
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -211,8 +258,12 @@
         assert ONOS6Port, "ONOS6Port not defined"
         assert ONOS7Port, "ONOS7Port not defined"
 
-        main.log.report( "Assigning switches to controllers" )
         main.case( "Assigning Controllers" )
+        main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+                                "and check that an ONOS node becomes the " +\
+                                "master of the device. Then manually assign" +\
+                                " mastership to specific ONOS nodes using" +\
+                                " 'device-role'"
         main.step( "Assign switches to controllers" )
 
         # TODO: rewrite this function to take lists of ips and ports?
@@ -244,16 +295,18 @@
                                     "not in the list of controllers s" +
                                     str( i ) + " is connecting to." )
                     mastershipCheck = main.FALSE
-        if mastershipCheck == main.TRUE:
-            main.log.report( "Switch mastership assigned correctly" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=mastershipCheck,
             onpass="Switch mastership assigned correctly",
             onfail="Switches not assigned correctly to controllers" )
+
+        main.step( "Assign mastership of switches to specific controllers" )
         # Manually assign mastership to the controller we want
         roleCall = main.TRUE
-        roleCheck = main.TRUE
+
+        ipList = [ ]
+        deviceList = []
         try:
             for i in range( 1, 29 ):  # switches 1 through 28
                 # set up correct variables:
@@ -297,16 +350,8 @@
                 # TODO: make this controller dynamic
                 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
                                                                   ip )
-                # Check assignment
-                master =  main.ONOScli1.getRole( deviceId ).get( 'master' )
-                if ip in master:
-                    roleCheck = roleCheck and main.TRUE
-                else:
-                    roleCheck = roleCheck and main.FALSE
-                    main.log.error( "Error, controller " + ip + " is not" +
-                                    " master " + "of device " +
-                                    str( deviceId ) + ". Master is " +
-                                    repr( master ) + "." )
+                ipList.append( ip )
+                deviceList.append( deviceId )
         except ( AttributeError, AssertionError ):
             main.log.exception( "Something is wrong with ONOS device view" )
             main.log.info( main.ONOScli1.devices() )
@@ -316,6 +361,24 @@
             onpass="Re-assigned switch mastership to designated controller",
             onfail="Something wrong with deviceRole calls" )
 
+        main.step( "Check mastership was correctly assigned" )
+        roleCheck = main.TRUE
+        # NOTE: This is due to the fact that device mastership change is not
+        #       atomic and is actually a multi step process
+        time.sleep( 5 )
+        for i in range( len( ipList ) ):
+            ip = ipList[i]
+            deviceId = deviceList[i]
+            # Check assignment
+            master = main.ONOScli1.getRole( deviceId ).get( 'master' )
+            if ip in master:
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+                main.log.error( "Error, controller " + ip + " is not" +
+                                " master " + "of device " +
+                                str( deviceId ) + ". Master is " +
+                                repr( master ) + "." )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=roleCheck,
@@ -332,8 +395,6 @@
         """
         Assign intents
         """
-        # FIXME: we must reinstall intents until we have a persistant
-        # datastore!
         import time
         import json
         assert numControllers, "numControllers not defined"
@@ -341,17 +402,23 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        main.log.report( "Adding host intents" )
+        # NOTE: we must reinstall intents until we have a persistant intent
+        #        datastore!
         main.case( "Adding host Intents" )
-
-        main.step( "Discovering  Hosts( Via pingall for now )" )
-        # FIXME: Once we have a host discovery mechanism, use that instead
+        main.caseExplaination = "Discover hosts by using pingall then " +\
+                                "assign predetermined host-to-host intents." +\
+                                " After installation, check that the intent" +\
+                                " is distributed to all nodes and the state" +\
+                                " is INSTALLED"
 
         # install onos-app-fwd
-        main.log.info( "Install reactive forwarding app" )
-        appResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+        main.step( "Install reactive forwarding app" )
+        installResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+        utilities.assert_equals( expect=main.TRUE, actual=installResults,
+                                 onpass="Install fwd successful",
+                                 onfail="Install fwd failed" )
 
-        # FIXME: add this to asserts
+        main.step( "Check app ids" )
         appCheck = main.TRUE
         threads = []
         for i in range( numControllers ):
@@ -364,13 +431,15 @@
         for t in threads:
             t.join()
             appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
         if appCheck != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
+        main.step( "Discovering Hosts( Via pingall for now )" )
+        # FIXME: Once we have a host discovery mechanism, use that instead
         # REACTIVE FWD test
         pingResult = main.FALSE
         for i in range(2):  # Retry if pingall fails first time
@@ -380,14 +449,23 @@
                 expect=main.TRUE,
                 actual=pingResult,
                 onpass="Reactive Pingall test passed",
-                onfail="Reactive Pingall failed, one or more ping pairs failed" )
+                onfail="Reactive Pingall failed, " +
+                       "one or more ping pairs failed" )
             time2 = time.time()
-            main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
-
+            main.log.info( "Time for pingall: %2f seconds" %
+                           ( time2 - time1 ) )
+        # timeout for fwd flows
+        time.sleep( 11 )
         # uninstall onos-app-fwd
-        main.log.info( "Uninstall reactive forwarding app" )
-        appResults = appResults and CLIs[0].deactivateApp( "org.onosproject.fwd" )
+        main.step( "Uninstall reactive forwarding app" )
+        uninstallResult = CLIs[0].deactivateApp( "org.onosproject.fwd" )
+        utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
+                                 onpass="Uninstall fwd successful",
+                                 onfail="Uninstall fwd failed" )
+
+        main.step( "Check app ids" )
         threads = []
+        appCheck2 = main.TRUE
         for i in range( numControllers ):
             t = main.Thread( target=CLIs[i].appToIDCheck,
                              name="appToIDCheck-" + str( i ),
@@ -397,18 +475,15 @@
 
         for t in threads:
             t.join()
-            appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
-        if appCheck != main.TRUE:
+            appCheck2 = appCheck2 and t.result
+        if appCheck2 != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
-        # timeout for fwd flows
-        time.sleep( 11 )
-
-        main.step( "Add host intents" )
+        main.step( "Add host intents via cli" )
         intentIds = []
         # TODO:  move the host numbers to params
         #        Maybe look at all the paths we ping?
@@ -451,7 +526,10 @@
                 except ( ValueError, TypeError ):
                     main.log.warn( repr( hosts ) )
                 hostResult = main.FALSE
-        # FIXME: DEBUG
+        utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+                                 onpass="Found a host id for each host",
+                                 onfail="Error looking up host ids" )
+
         intentStart = time.time()
         onosIds = main.ONOScli1.getAllIntentsId()
         main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -461,7 +539,6 @@
                 pass  # intent submitted is in onos
             else:
                 intentAddResult = False
-        # FIXME: DEBUG
         if intentAddResult:
             intentStop = time.time()
         else:
@@ -546,35 +623,49 @@
             main.log.exception( "Error parsing pending map" )
             main.log.error( repr( pendingMap ) )
 
-        intentAddResult = bool( pingResult and hostResult and intentAddResult
-                                and not missingIntents and installedCheck )
-        utilities.assert_equals(
-            expect=True,
-            actual=intentAddResult,
-            onpass="Pushed host intents to ONOS",
-            onfail="Error in pushing host intents to ONOS" )
+        intentAddResult = bool( intentAddResult and not missingIntents and
+                                installedCheck )
+        if not intentAddResult:
+            main.log.error( "Error in pushing host intents to ONOS" )
+
+        main.step( "Intent Anti-Entropy dispersion" )
         for i in range(100):
-            onosIds = main.ONOScli1.getAllIntentsId()
+            correct = True
             main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
-            main.log.info( "Intents in ONOS: " + str( sorted( onosIds ) ) )
-            if sorted(onosIds) == sorted(intentIds):
+            for cli in CLIs:
+                onosIds = []
+                ids = cli.getAllIntentsId()
+                onosIds.append( ids )
+                main.log.debug( "Intents in " + cli.name + ": " +
+                                str( sorted( onosIds ) ) )
+                if sorted( ids ) != sorted( intentIds ):
+                    # IF intents are missing
+                    correct = False
+                    break
+                else:
+                    intents = json.loads( cli.intents() )
+                    for intent in intents:
+                        if intent[ 'state' ] != "INSTALLED":
+                            correct = False
+                            break
+            if correct:
                 break
             else:
                 time.sleep(1)
-        # FIXME: DEBUG
         if not intentStop:
             intentStop = time.time()
+        global gossipTime
         gossipTime = intentStop - intentStart
         main.log.info( "It took about " + str( gossipTime ) +
-                        " seconds for all intents to appear on ONOS1" )
+                        " seconds for all intents to appear in each node" )
         # FIXME: make this time configurable/calculate based off of number of
         #        nodes and gossip rounds
         utilities.assert_greater_equals(
-                expect=30, actual=gossipTime,
+                expect=40, actual=gossipTime,
                 onpass="ECM anti-entropy for intents worked within " +
                        "expected time",
                 onfail="Intent ECM anti-entropy took too long" )
-        if gossipTime <= 30:
+        if gossipTime <= 40:
             intentAddResult = True
 
         if not intentAddResult or "key" in pendingMap:
@@ -677,9 +768,11 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        description = " Ping across added host intents"
-        main.log.report( description )
-        main.case( description )
+        main.case( "Verify connectivity by sendind traffic across Intents" )
+        main.caseExplaination = "Ping across added host intents to check " +\
+                                "functionality and check the state of " +\
+                                "the intent"
+        main.step( "Ping across added host intents" )
         PingResult = main.TRUE
         for i in range( 8, 18 ):
             ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -692,7 +785,7 @@
                 main.log.info( "Ping test passed!" )
                 # Don't set PingResult or you'd override failures
         if PingResult == main.FALSE:
-            main.log.report(
+            main.log.error(
                 "Intents have not been installed correctly, pings failed." )
             # TODO: pretty print
             main.log.warn( "ONOS1 intents: " )
@@ -704,17 +797,17 @@
                                            separators=( ',', ': ' ) ) )
             except ( ValueError, TypeError ):
                 main.log.warn( repr( tmpIntents ) )
-        if PingResult == main.TRUE:
-            main.log.report(
-                "Intents have been installed correctly and verified by pings" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=PingResult,
             onpass="Intents have been installed correctly and pings work",
             onfail="Intents have not been installed correctly, pings failed." )
 
-        installedCheck = True
-        if PingResult is not main.TRUE:
+        main.step( "Check Intent state" )
+        installedCheck = False
+        count = 0
+        while not installedCheck and count < 40:
+            installedCheck = True
             # Print the intent states
             intents = main.ONOScli1.intents()
             intentStates = []
@@ -730,71 +823,92 @@
                     intentStates.append( ( intentId, state ) )
             except ( ValueError, TypeError ):
                 main.log.exception( "Error parsing intents." )
+            # Print states
             intentStates.sort()
             for i, s in intentStates:
                 count += 1
                 main.log.info( "%-6s%-15s%-15s" %
                                ( str( count ), str( i ), str( s ) ) )
-            leaders = main.ONOScli1.leaders()
-            try:
-                if leaders:
-                    parsedLeaders = json.loads( leaders )
-                    main.log.warn( json.dumps( parsedLeaders,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # check for all intent partitions
-                    # check for election
-                    topics = []
-                    for i in range( 14 ):
-                        topics.append( "intent-partition-" + str( i ) )
-                    # FIXME: this should only be after we start the app
-                    topics.append( "org.onosproject.election" )
-                    main.log.debug( topics )
-                    ONOStopics = [ j['topic'] for j in parsedLeaders ]
-                    for topic in topics:
-                        if topic not in ONOStopics:
-                            main.log.error( "Error: " + topic +
-                                            " not in leaders" )
-                else:
-                    main.log.error( "leaders() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing leaders" )
-                main.log.error( repr( leaders ) )
-            partitions = main.ONOScli1.partitions()
-            try:
-                if partitions :
-                    parsedPartitions = json.loads( partitions )
-                    main.log.warn( json.dumps( parsedPartitions,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # TODO check for a leader in all paritions
-                    # TODO check for consistency among nodes
-                else:
-                    main.log.error( "partitions() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing partitions" )
-                main.log.error( repr( partitions ) )
-            pendingMap = main.ONOScli1.pendingMap()
-            try:
-                if pendingMap :
-                    parsedPending = json.loads( pendingMap )
-                    main.log.warn( json.dumps( parsedPending,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # TODO check something here?
-                else:
-                    main.log.error( "pendingMap() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing pending map" )
-                main.log.error( repr( pendingMap ) )
+            if not installedCheck:
+                time.sleep( 1 )
+                count += 1
+        utilities.assert_equals( expect=True, actual=installedCheck,
+                                 onpass="Intents are all INSTALLED",
+                                 onfail="Intents are not all in " +
+                                        "INSTALLED state" )
 
+        main.step( "Check leadership of topics" )
+        leaders = main.ONOScli1.leaders()
+        topicCheck = main.TRUE
+        try:
+            if leaders:
+                parsedLeaders = json.loads( leaders )
+                main.log.warn( json.dumps( parsedLeaders,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # check for all intent partitions
+                # check for election
+                # TODO: Look at Devices as topics now that it uses this system
+                topics = []
+                for i in range( 14 ):
+                    topics.append( "intent-partition-" + str( i ) )
+                # FIXME: this should only be after we start the app
+                # FIXME: topics.append( "org.onosproject.election" )
+                # Print leaders output
+                main.log.debug( topics )
+                ONOStopics = [ j['topic'] for j in parsedLeaders ]
+                for topic in topics:
+                    if topic not in ONOStopics:
+                        main.log.error( "Error: " + topic +
+                                        " not in leaders" )
+                        topicCheck = main.FALSE
+            else:
+                main.log.error( "leaders() returned None" )
+                topicCheck = main.FALSE
+        except ( ValueError, TypeError ):
+            topicCheck = main.FALSE
+            main.log.exception( "Error parsing leaders" )
+            main.log.error( repr( leaders ) )
+            # TODO: Check for a leader of these topics
+        utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+                                 onpass="intent Partitions is in leaders",
+                                 onfail="Some topics were lost " )
+        # Print partitions
+        partitions = main.ONOScli1.partitions()
+        try:
+            if partitions :
+                parsedPartitions = json.loads( partitions )
+                main.log.warn( json.dumps( parsedPartitions,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # TODO check for a leader in all paritions
+                # TODO check for consistency among nodes
+            else:
+                main.log.error( "partitions() returned None" )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing partitions" )
+            main.log.error( repr( partitions ) )
+        # Print Pending Map
+        pendingMap = main.ONOScli1.pendingMap()
+        try:
+            if pendingMap :
+                parsedPending = json.loads( pendingMap )
+                main.log.warn( json.dumps( parsedPending,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # TODO check something here?
+            else:
+                main.log.error( "pendingMap() returned None" )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing pending map" )
+            main.log.error( repr( pendingMap ) )
+        main.log.info( "Waiting 60 seconds to see if the state of " +
+                       "intents change" )
+        time.sleep( 60 )
         if not installedCheck:
-            main.log.info( "Waiting 60 seconds to see if the state of " +
-                           "intents change" )
-            time.sleep( 60 )
             # Print the intent states
             intents = main.ONOScli1.intents()
             intentStates = []
@@ -870,6 +984,39 @@
             except ( ValueError, TypeError ):
                 main.log.exception( "Error parsing pending map" )
                 main.log.error( repr( pendingMap ) )
+        # Print flowrules
+        main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+        main.step( "Wait a minute then ping again" )
+        # the wait is above
+        PingResult = main.TRUE
+        for i in range( 8, 18 ):
+            ping = main.Mininet1.pingHost( src="h" + str( i ),
+                                           target="h" + str( i + 10 ) )
+            PingResult = PingResult and ping
+            if ping == main.FALSE:
+                main.log.warn( "Ping failed between h" + str( i ) +
+                               " and h" + str( i + 10 ) )
+            elif ping == main.TRUE:
+                main.log.info( "Ping test passed!" )
+                # Don't set PingResult or you'd override failures
+        if PingResult == main.FALSE:
+            main.log.error(
+                "Intents have not been installed correctly, pings failed." )
+            # TODO: pretty print
+            main.log.warn( "ONOS1 intents: " )
+            try:
+                tmpIntents = main.ONOScli1.intents()
+                main.log.warn( json.dumps( json.loads( tmpIntents ),
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+            except ( ValueError, TypeError ):
+                main.log.warn( repr( tmpIntents ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=PingResult,
+            onpass="Intents have been installed correctly and pings work",
+            onfail="Intents have not been installed correctly, pings failed." )
 
     def CASE5( self, main ):
         """
@@ -885,7 +1032,6 @@
         # assumes that sts is already in you PYTHONPATH
         from sts.topology.teston_topology import TestONTopology
 
-        main.log.report( "Setting up and gathering data for current state" )
         main.case( "Setting up and gathering data for current state" )
         # The general idea for this test case is to pull the state of
         # ( intents,flows, topology,... ) from each ONOS node
@@ -933,7 +1079,7 @@
 
         for i in range( numControllers ):
             if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " roles" )
                 main.log.warn(
                     "ONOS" + str( i + 1 ) + " mastership response: " +
@@ -947,7 +1093,7 @@
 
         main.step( "Check for consistency in roles from each controller" )
         if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
-            main.log.report(
+            main.log.info(
                 "Switch roles are consistent across all ONOS nodes" )
         else:
             consistentMastership = False
@@ -995,7 +1141,7 @@
 
         for i in range( numControllers ):
             if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " intents" )
                 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
                                repr( ONOSIntents[ i ] ) )
@@ -1008,34 +1154,63 @@
 
         main.step( "Check for consistency in Intents from each controller" )
         if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
-            main.log.report( "Intents are consistent across all ONOS " +
+            main.log.info( "Intents are consistent across all ONOS " +
                              "nodes" )
         else:
             consistentIntents = False
-            main.log.report( "Intents not consistent" )
+            main.log.error( "Intents not consistent" )
         utilities.assert_equals(
             expect=True,
             actual=consistentIntents,
             onpass="Intents are consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of intents" )
 
+        if intentsResults:
+            # Try to make it easy to figure out what is happening
+            #
+            # Intent      ONOS1      ONOS2    ...
+            #  0x01     INSTALLED  INSTALLING
+            #  ...        ...         ...
+            #  ...        ...         ...
+            title = "   Id"
+            for n in range( numControllers ):
+                title += " " * 10 + "ONOS" + str( n + 1 )
+            main.log.warn( title )
+            # get all intent keys in the cluster
+            keys = []
+            for nodeStr in ONOSIntents:
+                node = json.loads( nodeStr )
+                for intent in node:
+                    keys.append( intent.get( 'id' ) )
+            keys = set( keys )
+            for key in keys:
+                row = "%-13s" % key
+                for nodeStr in ONOSIntents:
+                    node = json.loads( nodeStr )
+                    for intent in node:
+                        if intent.get( 'id', "Error" ) == key:
+                            row += "%-15s" % intent.get( 'state' )
+                main.log.warn( row )
+            # End table view
+
         if intentsResults and not consistentIntents:
+            # print the json objects
             n = len(ONOSIntents)
-            main.log.warn( "ONOS" + str( n ) + " intents: " )
-            main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
-                                       sort_keys=True,
-                                       indent=4,
-                                       separators=( ',', ': ' ) ) )
+            main.log.debug( "ONOS" + str( n ) + " intents: " )
+            main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+                                        sort_keys=True,
+                                        indent=4,
+                                        separators=( ',', ': ' ) ) )
             for i in range( numControllers ):
                 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
-                    main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
-                    main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
+                    main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
+                    main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
+                                                sort_keys=True,
+                                                indent=4,
+                                                separators=( ',', ': ' ) ) )
                 else:
-                    main.log.warn( nodes[ i ].name + " intents match ONOS" +
-                                   str( n ) + " intents" )
+                    main.log.debug( nodes[ i ].name + " intents match ONOS" +
+                                    str( n ) + " intents" )
         elif intentsResults and consistentIntents:
             intentCheck = main.TRUE
             intentState = ONOSIntents[ 0 ]
@@ -1067,7 +1242,7 @@
         for i in range( numControllers ):
             num = str( i + 1 )
             if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
-                main.log.report( "Error in getting ONOS" + num + " flows" )
+                main.log.error( "Error in getting ONOS" + num + " flows" )
                 main.log.warn( "ONOS" + num + " flows response: " +
                                repr( ONOSFlows[ i ] ) )
                 flowsResults = False
@@ -1091,7 +1266,7 @@
         main.step( "Check for consistency in Flows from each controller" )
         tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
         if all( tmp ):
-            main.log.report( "Flow count is consistent across all ONOS nodes" )
+            main.log.info( "Flow count is consistent across all ONOS nodes" )
         else:
             consistentFlows = False
         utilities.assert_equals(
@@ -1219,6 +1394,14 @@
         for t in threads:
             t.join()
             ports.append( t.result )
+            try:
+                # FIXME: DEBUG
+                main.log.debug( json.dumps( json.loads( t.result ),
+                                            sort_keys=True,
+                                            indent=4,
+                                            separators=( ',', ': ' ) ) )
+            except:
+                main.log.debug( repr( t.result ) )
         links = []
         threads = []
         for i in range( numControllers ):
@@ -1246,6 +1429,7 @@
         # Compare json objects for hosts and dataplane clusters
 
         # hosts
+        main.step( "Host view is consistent across ONOS nodes" )
         consistentHostsResult = main.TRUE
         for controller in range( len( hosts ) ):
             controllerStr = str( controller + 1 )
@@ -1253,14 +1437,14 @@
                 if hosts[ controller ] == hosts[ 0 ]:
                     continue
                 else:  # hosts not consistent
-                    main.log.report( "hosts from ONOS" +
+                    main.log.error( "hosts from ONOS" +
                                      controllerStr +
                                      " is inconsistent with ONOS1" )
                     main.log.warn( repr( hosts[ controller ] ) )
                     consistentHostsResult = main.FALSE
 
             else:
-                main.log.report( "Error in getting ONOS hosts from ONOS" +
+                main.log.error( "Error in getting ONOS hosts from ONOS" +
                                  controllerStr )
                 consistentHostsResult = main.FALSE
                 main.log.warn( "ONOS" + controllerStr +
@@ -1272,11 +1456,12 @@
             onpass="Hosts view is consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of hosts" )
 
+        main.step( "Each host has an IP address" )
         ipResult = main.TRUE
         for controller in range( 0, len( hosts ) ):
             controllerStr = str( controller + 1 )
             for host in hosts[ controller ]:
-                if not host.get( 'ips', [ ] ):
+                if not host.get( 'ipAddresses', [ ] ):
                     main.log.error( "DEBUG:Error with host ips on controller" +
                                     controllerStr + ": " + str( host ) )
                     ipResult = main.FALSE
@@ -1287,6 +1472,7 @@
             onfail="The ip of at least one host is missing" )
 
         # Strongly connected clusters of devices
+        main.step( "Cluster view is consistent across ONOS nodes" )
         consistentClustersResult = main.TRUE
         for controller in range( len( clusters ) ):
             controllerStr = str( controller + 1 )
@@ -1294,12 +1480,12 @@
                 if clusters[ controller ] == clusters[ 0 ]:
                     continue
                 else:  # clusters not consistent
-                    main.log.report( "clusters from ONOS" + controllerStr +
+                    main.log.error( "clusters from ONOS" + controllerStr +
                                      " is inconsistent with ONOS1" )
                     consistentClustersResult = main.FALSE
 
             else:
-                main.log.report( "Error in getting dataplane clusters " +
+                main.log.error( "Error in getting dataplane clusters " +
                                  "from ONOS" + controllerStr )
                 consistentClustersResult = main.FALSE
                 main.log.warn( "ONOS" + controllerStr +
@@ -1311,6 +1497,7 @@
             onpass="Clusters view is consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of clusters" )
         # there should always only be one cluster
+        main.step( "Cluster view correct across ONOS nodes" )
         try:
             numClusters = len( json.loads( clusters[ 0 ] ) )
         except ( ValueError, TypeError ):
@@ -1381,30 +1568,43 @@
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
 
-        finalAssert = main.TRUE
-        finalAssert = ( finalAssert and topoResult and flowCheck
-                        and intentCheck and consistentMastership
-                        and mastershipCheck and rolesNotNull )
-        utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
-                                 onpass="State check successful",
-                                 onfail="State check NOT successful" )
-
     def CASE6( self, main ):
         """
         The Failure case.
         """
+        import time
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        main.log.report( "Restart entire ONOS cluster" )
+
+        # Reset non-persistent variables
+        try:
+            iCounterValue = 0
+        except NameError:
+            main.log.error( "iCounterValue not defined, setting to 0" )
+            iCounterValue = 0
+
         main.case( "Restart entire ONOS cluster" )
+        # FIXME: DEBUG
+        main.step( "Start Packet Capture MN" )
+        main.Mininet2.startTcpdump(
+            str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+            + "-MN.pcap",
+            intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+            port=main.params[ 'MNtcpdump' ][ 'port' ] )
+        # FIXME: DEBUG
+
         main.step( "Killing ONOS nodes" )
         killResults = main.TRUE
+        killTime = time.time()
         for node in nodes:
             killed = main.ONOSbench.onosKill( node.ip_address )
             killResults = killResults and killed
+        utilities.assert_equals( expect=main.TRUE, actual=killResults,
+                                 onpass="ONOS nodes killed",
+                                 onfail="ONOS kill unsuccessful" )
 
         main.step( "Checking if ONOS is up yet" )
         for i in range( 2 ):
@@ -1412,10 +1612,13 @@
             for node in nodes:
                 started = main.ONOSbench.isup( node.ip_address )
                 if not started:
-                    main.log.report( node.name + " didn't start!" )
+                    main.log.error( node.name + " didn't start!" )
                 onosIsupResult = onosIsupResult and started
             if onosIsupResult == main.TRUE:
                 break
+        utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+                                 onpass="ONOS restarted",
+                                 onfail="ONOS restart NOT successful" )
 
         main.log.step( "Starting ONOS CLI sessions" )
         cliResults = main.TRUE
@@ -1430,11 +1633,32 @@
         for t in threads:
             t.join()
             cliResults = cliResults and t.result
+        utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                 onpass="ONOS cli started",
+                                 onfail="ONOS clis did not restart" )
 
-        caseResults = main.TRUE and onosIsupResult and cliResults
-        utilities.assert_equals( expect=main.TRUE, actual=caseResults,
-                                 onpass="ONOS restart successful",
-                                 onfail="ONOS restart NOT successful" )
+        # Grab the time of restart so we chan check how long the gossip
+        # protocol has had time to work
+        main.restartTime = time.time() - killTime
+        main.log.debug( "Restart time: " + str( main.restartTime ) )
+
+        # FIXME: revisit test plan for election with madan
+        # Rerun for election on restarted nodes
+        runResults = main.TRUE
+        for cli in CLIs:
+            run = CLIs[0].electionTestRun()
+            if run != main.TRUE:
+                main.log.error( "Error running for election on " + cli.name )
+            runResults = runResults and run
+        utilities.assert_equals( expect=main.TRUE, actual=runResults,
+                                 onpass="Reran for election",
+                                 onfail="Failed to rerun for election" )
+
+        # TODO: Make this configurable
+        time.sleep( 60 )
+        main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+        main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+        main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
 
     def CASE7( self, main ):
         """
@@ -1468,6 +1692,7 @@
             onpass="Each device has a master",
             onfail="Some devices don't have a master assigned" )
 
+        main.step( "Read device roles from ONOS" )
         ONOSMastership = []
         mastershipCheck = main.FALSE
         consistentMastership = True
@@ -1486,7 +1711,7 @@
 
         for i in range( numControllers ):
             if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " roles" )
                 main.log.warn(
                     "ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1500,7 +1725,7 @@
 
         main.step( "Check for consistency in roles from each controller" )
         if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
-            main.log.report(
+            main.log.info(
                 "Switch roles are consistent across all ONOS nodes" )
         else:
             consistentMastership = False
@@ -1522,6 +1747,7 @@
         elif rolesResults and not consistentMastership:
             mastershipCheck = main.TRUE
 
+        '''
         description2 = "Compare switch roles from before failure"
         main.step( description2 )
         try:
@@ -1547,15 +1773,13 @@
             else:
                 main.log.warn( "Mastership of switch %s changed" % switchDPID )
                 mastershipCheck = main.FALSE
-        if mastershipCheck == main.TRUE:
-            main.log.report( "Mastership of Switches was not changed" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=mastershipCheck,
             onpass="Mastership of Switches was not changed",
             onfail="Mastership of some switches changed" )
+        '''
         # NOTE: we expect mastership to change on controller failure
-        mastershipCheck = mastershipCheck and consistentMastership
 
         main.step( "Get the intents and compare across all nodes" )
         ONOSIntents = []
@@ -1577,7 +1801,7 @@
 
         for i in range( numControllers ):
             if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " intents" )
                 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
                                repr( ONOSIntents[ i ] ) )
@@ -1590,10 +1814,38 @@
 
         main.step( "Check for consistency in Intents from each controller" )
         if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
-            main.log.report( "Intents are consistent across all ONOS " +
+            main.log.info( "Intents are consistent across all ONOS " +
                              "nodes" )
         else:
             consistentIntents = False
+
+        # Try to make it easy to figure out what is happening
+        #
+        # Intent      ONOS1      ONOS2    ...
+        #  0x01     INSTALLED  INSTALLING
+        #  ...        ...         ...
+        #  ...        ...         ...
+        title = "   ID"
+        for n in range( numControllers ):
+            title += " " * 10 + "ONOS" + str( n + 1 )
+        main.log.warn( title )
+        # get all intent keys in the cluster
+        keys = []
+        for nodeStr in ONOSIntents:
+            node = json.loads( nodeStr )
+            for intent in node:
+                keys.append( intent.get( 'id' ) )
+        keys = set( keys )
+        for key in keys:
+            row = "%-13s" % key
+            for nodeStr in ONOSIntents:
+                node = json.loads( nodeStr )
+                for intent in node:
+                    if intent.get( 'id' ) == key:
+                        row += "%-15s" % intent.get( 'state' )
+            main.log.warn( row )
+        # End table view
+
         utilities.assert_equals(
             expect=True,
             actual=consistentIntents,
@@ -1630,22 +1882,41 @@
         main.step( "Compare current intents with intents before the failure" )
         # NOTE: this requires case 5 to pass for intentState to be set.
         #      maybe we should stop the test if that fails?
-        sameIntents = main.TRUE
+        sameIntents = main.FALSE
         if intentState and intentState == ONOSIntents[ 0 ]:
             sameIntents = main.TRUE
-            main.log.report( "Intents are consistent with before failure" )
+            main.log.info( "Intents are consistent with before failure" )
         # TODO: possibly the states have changed? we may need to figure out
         #       what the acceptable states are
-        else:
+        elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+            sameIntents = main.TRUE
             try:
-                main.log.warn( "ONOS intents: " )
-                main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
-                                           sort_keys=True, indent=4,
-                                           separators=( ',', ': ' ) ) )
+                before = json.loads( intentState )
+                after = json.loads( ONOSIntents[ 0 ] )
+                for intent in before:
+                    if intent not in after:
+                        sameIntents = main.FALSE
+                        main.log.debug( "Intent is not currently in ONOS " +\
+                                        "(at least in the same form):" )
+                        main.log.debug( json.dumps( intent ) )
             except ( ValueError, TypeError ):
                 main.log.exception( "Exception printing intents" )
-                main.log.warn( repr( ONOSIntents[0] ) )
-            sameIntents = main.FALSE
+                main.log.debug( repr( ONOSIntents[0] ) )
+                main.log.debug( repr( intentState ) )
+        if sameIntents == main.FALSE:
+            try:
+                main.log.debug( "ONOS intents before: " )
+                main.log.debug( json.dumps( json.loads( intentState ),
+                                            sort_keys=True, indent=4,
+                                            separators=( ',', ': ' ) ) )
+                main.log.debug( "Current ONOS intents: " )
+                main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+                                            sort_keys=True, indent=4,
+                                            separators=( ',', ': ' ) ) )
+            except ( ValueError, TypeError ):
+                main.log.exception( "Exception printing intents" )
+                main.log.debug( repr( ONOSIntents[0] ) )
+                main.log.debug( repr( intentState ) )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=sameIntents,
@@ -1668,16 +1939,14 @@
             if FlowTables == main.FALSE:
                 main.log.info( "Differences in flow table for switch: s" +
                                str( i + 1 ) )
-        if FlowTables == main.TRUE:
-            main.log.report( "No changes were found in the flow tables" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=FlowTables,
             onpass="No changes were found in the flow tables",
             onfail="Changes were found in the flow tables" )
 
-        main.step( "Check the continuous pings to ensure that no packets " +
-                   "were dropped during component failure" )
+        # main.step( "Check the continuous pings to ensure that no packets " +
+        #            "were dropped during component failure" )
         main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
                                 main.params[ 'TESTONIP' ] )
         LossInPings = main.FALSE
@@ -1695,16 +1964,18 @@
             main.log.info( "There are multiple mininet process running" )
         elif LossInPings == main.FALSE:
             main.log.info( "No Loss in the pings" )
-            main.log.report( "No loss of dataplane connectivity" )
-        utilities.assert_equals(
-            expect=main.FALSE,
-            actual=LossInPings,
-            onpass="No Loss of connectivity",
-            onfail="Loss of dataplane connectivity detected" )
+            main.log.info( "No loss of dataplane connectivity" )
+        # utilities.assert_equals(
+        #     expect=main.FALSE,
+        #     actual=LossInPings,
+        #     onpass="No Loss of connectivity",
+        #     onfail="Loss of dataplane connectivity detected" )
+
         # NOTE: Since intents are not persisted with IntnentStore,
         #       we expect loss in dataplane connectivity
         LossInPings = main.FALSE
 
+        main.step( "Leadership Election is still functional" )
         # Test of LeadershipElection
         leaderList = []
         leaderResult = main.TRUE
@@ -1712,40 +1983,26 @@
             leaderN = cli.electionTestLeader()
             leaderList.append( leaderN )
             if leaderN == main.FALSE:
-                # error in  response
-                main.log.report( "Something is wrong with " +
+                # error in response
+                main.log.error( "Something is wrong with " +
                                  "electionTestLeader function, check the" +
                                  " error logs" )
                 leaderResult = main.FALSE
             elif leaderN is None:
-                main.log.report( cli.name +
-                                 " shows no leader for the election-app was" +
-                                 " elected after the old one died" )
+                main.log.error( cli.name +
+                                 " shows no leader for the election-app." )
                 leaderResult = main.FALSE
         if len( set( leaderList ) ) != 1:
             leaderResult = main.FALSE
             main.log.error(
                 "Inconsistent view of leader for the election test app" )
             # TODO: print the list
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a new " +
-                             "leader was re-elected if applicable )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
             onpass="Leadership election passed",
             onfail="Something went wrong with Leadership election" )
 
-        result = ( mastershipCheck and intentCheck and FlowTables and
-                   ( not LossInPings ) and rolesNotNull and leaderResult )
-        result = int( result )
-        if result == main.TRUE:
-            main.log.report( "Constant State Tests Passed" )
-        utilities.assert_equals( expect=main.TRUE, actual=result,
-                                 onpass="Constant State Tests Passed",
-                                 onfail="Constant state tests failed" )
-
     def CASE8( self, main ):
         """
         Compare topo
@@ -1763,15 +2020,24 @@
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
 
-        description = "Compare ONOS Topology view to Mininet topology"
-        main.case( description )
-        main.log.report( description )
+        main.case( "Compare ONOS Topology view to Mininet topology" )
+        main.caseExplaination = "Compare topology objects between Mininet" +\
+                                " and ONOS"
         main.step( "Create TestONTopology object" )
-        ctrls = []
-        for node in nodes:
-            temp = ( node, node.name, node.ip_address, 6633 )
-            ctrls.append( temp )
-        MNTopo = TestONTopology( main.Mininet1, ctrls )
+        try:
+            ctrls = []
+            for node in nodes:
+                temp = ( node, node.name, node.ip_address, 6633 )
+                ctrls.append( temp )
+            MNTopo = TestONTopology( main.Mininet1, ctrls )
+        except Exception:
+            objResult = main.FALSE
+        else:
+            objResult = main.TRUE
+        utilities.assert_equals( expect=main.TRUE, actual=objResult,
+                                 onpass="Created TestONTopology object",
+                                 onfail="Exception while creating " +
+                                        "TestONTopology object" )
 
         main.step( "Comparing ONOS topology to MN" )
         devicesResults = main.TRUE
@@ -1784,11 +2050,12 @@
         main.step( "Collecting topology information from ONOS" )
         startTime = time.time()
         # Give time for Gossip to work
-        while topoResult == main.FALSE and elapsed < 60:
+        while topoResult == main.FALSE and elapsed < 120:
             count += 1
             if count > 1:
                 # TODO: Deprecate STS usage
                 MNTopo = TestONTopology( main.Mininet1, ctrls )
+                time.sleep( 1 )
             cliStart = time.time()
             devices = []
             threads = []
@@ -1822,9 +2089,9 @@
             for controller in range( 0, len( hosts ) ):
                 controllerStr = str( controller + 1 )
                 for host in hosts[ controller ]:
-                    if host is None or host.get( 'ips', [] ) == []:
+                    if host is None or host.get( 'ipAddresses', [] ) == []:
                         main.log.error(
-                            "DEBUG:Error with host ips on controller" +
+                            "DEBUG:Error with host ipAddresses on controller" +
                             controllerStr + ": " + str( host ) )
                         ipResult = main.FALSE
             ports = []
@@ -1839,6 +2106,14 @@
             for t in threads:
                 t.join()
                 ports.append( t.result )
+                try:
+                    # FIXME: DEBUG
+                    main.log.debug( json.dumps( json.loads( t.result ),
+                                                sort_keys=True,
+                                                indent=4,
+                                                separators=( ',', ': ' ) ) )
+                except:
+                    main.log.debug( repr( t.result ) )
             links = []
             threads = []
             for i in range( numControllers ):
@@ -1866,6 +2141,7 @@
 
             elapsed = time.time() - startTime
             cliTime = time.time() - cliStart
+            print "Elapsed time: " + str( elapsed )
             print "CLI time: " + str( cliTime )
 
             for controller in range( numControllers ):
@@ -1930,6 +2206,7 @@
             # Compare json objects for hosts and dataplane clusters
 
             # hosts
+            main.step( "Hosts view is consistent across all ONOS nodes" )
             consistentHostsResult = main.TRUE
             for controller in range( len( hosts ) ):
                 controllerStr = str( controller + 1 )
@@ -1937,13 +2214,13 @@
                     if hosts[ controller ] == hosts[ 0 ]:
                         continue
                     else:  # hosts not consistent
-                        main.log.report( "hosts from ONOS" + controllerStr +
+                        main.log.error( "hosts from ONOS" + controllerStr +
                                          " is inconsistent with ONOS1" )
                         main.log.warn( repr( hosts[ controller ] ) )
                         consistentHostsResult = main.FALSE
 
                 else:
-                    main.log.report( "Error in getting ONOS hosts from ONOS" +
+                    main.log.error( "Error in getting ONOS hosts from ONOS" +
                                      controllerStr )
                     consistentHostsResult = main.FALSE
                     main.log.warn( "ONOS" + controllerStr +
@@ -1956,6 +2233,7 @@
                 onfail="ONOS nodes have different views of hosts" )
 
             # Strongly connected clusters of devices
+            main.step( "Clusters view is consistent across all ONOS nodes" )
             consistentClustersResult = main.TRUE
             for controller in range( len( clusters ) ):
                 controllerStr = str( controller + 1 )
@@ -1963,13 +2241,13 @@
                     if clusters[ controller ] == clusters[ 0 ]:
                         continue
                     else:  # clusters not consistent
-                        main.log.report( "clusters from ONOS" +
+                        main.log.error( "clusters from ONOS" +
                                          controllerStr +
                                          " is inconsistent with ONOS1" )
                         consistentClustersResult = main.FALSE
 
                 else:
-                    main.log.report( "Error in getting dataplane clusters " +
+                    main.log.error( "Error in getting dataplane clusters " +
                                      "from ONOS" + controllerStr )
                     consistentClustersResult = main.FALSE
                     main.log.warn( "ONOS" + controllerStr +
@@ -1981,6 +2259,8 @@
                 onpass="Clusters view is consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of clusters" )
             # there should always only be one cluster
+            main.step( "Topology view is correct and consistent across all " +
+                       "ONOS nodes" )
             try:
                 numClusters = len( json.loads( clusters[ 0 ] ) )
             except ( ValueError, TypeError ):
@@ -2011,12 +2291,11 @@
         utilities.assert_equals( expect=main.TRUE, actual=topoResult,
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
-        if topoResult == main.TRUE:
-            main.log.report( "ONOS topology view matches Mininet topology" )
 
         # FIXME: move this to an ONOS state case
         main.step( "Checking ONOS nodes" )
         nodesOutput = []
+        nodeResults = main.TRUE
         threads = []
         for i in range( numControllers ):
             t = main.Thread( target=CLIs[i].nodes,
@@ -2033,9 +2312,10 @@
             try:
                 current = json.loads( i )
                 for node in current:
+                    currentResult = main.FALSE
                     if node['ip'] in ips:  # node in nodes() output is in cell
                         if node['state'] == 'ACTIVE':
-                            pass  # as it should be
+                            currentResult = main.TRUE
                         else:
                             main.log.error( "Error in ONOS node availability" )
                             main.log.error(
@@ -2044,9 +2324,13 @@
                                                 indent=4,
                                                 separators=( ',', ': ' ) ) )
                             break
+                    nodeResults = nodeResults and currentResult
             except ( ValueError, TypeError ):
                 main.log.error( "Error parsing nodes output" )
                 main.log.warn( repr( i ) )
+        utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
 
     def CASE9( self, main ):
         """
@@ -2064,7 +2348,6 @@
 
         description = "Turn off a link to ensure that Link Discovery " +\
                       "is working properly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Kill Link between s3 and s28" )
@@ -2093,7 +2376,6 @@
 
         description = "Restore a link to ensure that Link Discovery is " + \
                       "working properly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Bring link between s3 and s28 back up" )
@@ -2121,14 +2403,13 @@
         switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
 
         description = "Killing a switch to ensure it is discovered correctly"
-        main.log.report( description )
         main.case( description )
         switch = main.params[ 'kill' ][ 'switch' ]
         switchDPID = main.params[ 'kill' ][ 'dpid' ]
 
         # TODO: Make this switch parameterizable
         main.step( "Kill " + switch )
-        main.log.report( "Deleting " + switch )
+        main.log.info( "Deleting " + switch )
         main.Mininet1.delSwitch( switch )
         main.log.info( "Waiting " + str( switchSleep ) +
                        " seconds for switch down to be discovered" )
@@ -2167,11 +2448,9 @@
         switchDPID = main.params[ 'kill' ][ 'dpid' ]
         links = main.params[ 'kill' ][ 'links' ].split()
         description = "Adding a switch to ensure it is discovered correctly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Add back " + switch )
-        main.log.report( "Adding back " + switch )
         main.Mininet1.addSwitch( switch, dpid=switchDPID )
         for peer in links:
             main.Mininet1.addLink( switch, peer )
@@ -2220,9 +2499,7 @@
         colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
                    'blue': '\033[94m', 'green': '\033[92m',
                    'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
-        description = "Test Cleanup"
-        main.log.report( description )
-        main.case( description )
+        main.case( "Test Cleanup" )
         main.step( "Killing tcpdumps" )
         main.Mininet2.stopTcpdump()
 
@@ -2269,26 +2546,84 @@
         time.sleep( 10 )
 
         main.step( "Stopping Mininet" )
-        main.Mininet1.stopNet()
+        mnResult = main.Mininet1.stopNet()
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet stopped",
+                                 onfail="MN cleanup NOT successful" )
 
         main.step( "Checking ONOS Logs for errors" )
         for node in nodes:
             print colors[ 'purple' ] + "Checking logs for errors on " + \
                 node.name + ":" + colors[ 'end' ]
-            print main.ONOSbench.checkLogs( node.ip_address )
+            print main.ONOSbench.checkLogs( node.ip_address, restart=True )
 
         main.step( "Packing and rotating pcap archives" )
         os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
 
-        # TODO: actually check something here
-        utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
-                                 onpass="Test cleanup successful",
-                                 onfail="Test cleanup NOT successful" )
+        try:
+            timerLog = open( main.logdir + "/Timers.csv", 'w')
+            # Overwrite with empty line and close
+            labels = "Gossip Intents, Restart"
+            data = str( gossipTime ) + ", " + str( main.restartTime )
+            timerLog.write( labels + "\n" + data )
+            timerLog.close()
+        except NameError, e:
+            main.log.exception(e)
 
     def CASE14( self, main ):
         """
         start election app on all onos nodes
         """
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+
+        main.case("Start Leadership Election app")
+        main.step( "Install leadership election app" )
+        appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=appResult,
+            onpass="Election app installed",
+            onfail="Something went wrong with installing Leadership election" )
+
+        main.step( "Run for election on each node" )
+        leaderResult = main.TRUE
+        leaders = []
+        for cli in CLIs:
+            cli.electionTestRun()
+        for cli in CLIs:
+            leader = cli.electionTestLeader()
+            if leader is None or leader == main.FALSE:
+                main.log.error( cli.name + ": Leader for the election app " +
+                                 "should be an ONOS node, instead got '" +
+                                 str( leader ) + "'" )
+                leaderResult = main.FALSE
+            leaders.append( leader )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=leaderResult,
+            onpass="Successfully ran for leadership",
+            onfail="Failed to run for leadership" )
+
+        main.step( "Check that each node shows the same leader" )
+        sameLeader = main.TRUE
+        if len( set( leaders ) ) != 1:
+            sameLeader = main.FALSE
+            main.log.error( "Results of electionTestLeader is order of CLIs:" +
+                            str( leaders ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=sameLeader,
+            onpass="Leadership is consistent for the election topic",
+            onfail="Nodes have different leaders" )
+
+    def CASE15( self, main ):
+        """
+        Check that Leadership Election is still functional
+        """
         import time
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
@@ -2297,51 +2632,46 @@
         assert nodes, "nodes not defined"
 
         leaderResult = main.TRUE
-        main.log.info( "Install leadership election app" )
-        main.ONOScli1.activateApp( "org.onosproject.election" )
+        description = "Check that Leadership Election is still functional"
+        main.case( description )
+        # NOTE: Need to re-run since being a canidate is not persistant
+        main.step( "Run for election on each node" )
+        leaderResult = main.TRUE
         leaders = []
         for cli in CLIs:
+            cli.electionTestRun()
+        for cli in CLIs:
             leader = cli.electionTestLeader()
             if leader is None or leader == main.FALSE:
-                main.log.report( cli.name + ": Leader for the election app " +
+                main.log.error( cli.name + ": Leader for the election app " +
                                  "should be an ONOS node, instead got '" +
                                  str( leader ) + "'" )
                 leaderResult = main.FALSE
             leaders.append( leader )
-        if len( set( leaders ) ) != 1:
-            leaderResult = main.FALSE
-            main.log.error( "Results of electionTestLeader is order of CLIs:" +
-                            str( leaders ) )
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a leader " +
-                             "was elected )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
-            onpass="Leadership election passed",
-            onfail="Something went wrong with Leadership election" )
+            onpass="Successfully ran for leadership",
+            onfail="Failed to run for leadership" )
 
-    def CASE15( self, main ):
-        """
-        Check that Leadership Election is still functional
-        """
-        assert numControllers, "numControllers not defined"
-        assert main, "main not defined"
-        assert utilities.assert_equals, "utilities.assert_equals not defined"
-        assert CLIs, "CLIs not defined"
-        assert nodes, "nodes not defined"
+        main.step( "Check that each node shows the same leader" )
+        sameLeader = main.TRUE
+        if len( set( leaders ) ) != 1:
+            sameLeader = main.FALSE
+            main.log.error( "Results of electionTestLeader is order of CLIs:" +
+                            str( leaders ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=sameLeader,
+            onpass="Leadership is consistent for the election topic",
+            onfail="Nodes have different leaders" )
 
-        leaderResult = main.TRUE
-        description = "Check that Leadership Election is still functional"
-        main.log.report( description )
-        main.case( description )
         main.step( "Find current leader and withdraw" )
         leader = main.ONOScli1.electionTestLeader()
         # do some sanity checking on leader before using it
         withdrawResult = main.FALSE
         if leader is None or leader == main.FALSE:
-            main.log.report(
+            main.log.error(
                 "Leader for the election app should be an ONOS node," +
                 "instead got '" + str( leader ) + "'" )
             leaderResult = main.FALSE
@@ -2357,8 +2687,8 @@
         utilities.assert_equals(
             expect=main.TRUE,
             actual=withdrawResult,
-            onpass="App was withdrawn from election",
-            onfail="App was not withdrawn from election" )
+            onpass="Node was withdrawn from election",
+            onfail="Node was not withdrawn from election" )
 
         main.step( "Make sure new leader is elected" )
         # FIXME: use threads
@@ -2367,19 +2697,20 @@
             leaderN = cli.electionTestLeader()
             leaderList.append( leaderN )
             if leaderN == leader:
-                main.log.report(  cli.name + " still sees " + str( leader ) +
+                main.log.error(  cli.name + " still sees " + str( leader ) +
                                   " as leader after they withdrew" )
                 leaderResult = main.FALSE
             elif leaderN == main.FALSE:
                 # error in  response
                 # TODO: add check for "Command not found:" in the driver, this
                 #       means the app isn't loaded
-                main.log.report( "Something is wrong with " +
+                main.log.error( "Something is wrong with " +
                                  "electionTestLeader function, " +
                                  "check the error logs" )
                 leaderResult = main.FALSE
             elif leaderN is None:
                 # node may not have recieved the event yet
+                time.sleep(7)
                 leaderN = cli.electionTestLeader()
                 leaderList.pop()
                 leaderList.append( leaderN )
@@ -2390,17 +2721,12 @@
                            "' as the leader" )
             consistentLeader = main.TRUE
         else:
-            main.log.report(
+            main.log.error(
                 "Inconsistent responses for leader of Election-app:" )
             for n in range( len( leaderList ) ):
-                main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+                main.log.error( "ONOS" + str( n + 1 ) + " response: " +
                                  str( leaderList[ n ] ) )
         leaderResult = leaderResult and consistentLeader
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a new " +
-                             "leader was elected when the old leader " +
-                             "resigned )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
@@ -2418,18 +2744,1110 @@
             actual=runResult,
             onpass="App re-ran for election",
             onfail="App failed to run for election" )
-        if consistentLeader == main.TRUE:
-            afterRun = main.ONOScli1.electionTestLeader()
-            # verify leader didn't just change
-            if afterRun == leaderList[ 0 ]:
-                leaderResult = main.TRUE
-            else:
-                leaderResult = main.FALSE
-        # TODO: assert on  run and withdraw results?
+
+        main.step( "Leader did not change when old leader re-ran" )
+        afterRun = main.ONOScli1.electionTestLeader()
+        # verify leader didn't just change
+        if afterRun == leaderList[ 0 ]:
+            afterResult = main.TRUE
+        else:
+            afterResult = main.FALSE
 
         utilities.assert_equals(
             expect=main.TRUE,
-            actual=leaderResult,
-            onpass="Leadership election passed",
+            actual=afterResult,
+            onpass="Old leader successfully re-ran for election",
             onfail="Something went wrong with Leadership election after " +
                    "the old leader re-ran for election" )
+
+    def CASE16( self, main ):
+        """
+        Install Distributed Primitives app
+        """
+        import time
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+
+        # Variables for the distributed primitives tests
+        global pCounterName
+        global iCounterName
+        global pCounterValue
+        global iCounterValue
+        global onosSet
+        global onosSetName
+        pCounterName = "TestON-Partitions"
+        iCounterName = "TestON-inMemory"
+        pCounterValue = 0
+        iCounterValue = 0
+        onosSet = set([])
+        onosSetName = "TestON-set"
+
+        description = "Install Primitives app"
+        main.case( description )
+        main.step( "Install Primitives app" )
+        appName = "org.onosproject.distributedprimitives"
+        appResults = CLIs[0].activateApp( appName )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=appResults,
+                                 onpass="Primitives app activated",
+                                 onfail="Primitives app not activated" )
+        time.sleep( 5 )  # To allow all nodes to activate
+
+    def CASE17( self, main ):
+        """
+        Check for basic functionality with distributed primitives
+        """
+        # Make sure variables are defined/set
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+        assert pCounterName, "pCounterName not defined"
+        assert iCounterName, "iCounterName not defined"
+        assert onosSetName, "onosSetName not defined"
+        # NOTE: assert fails if value is 0/None/Empty/False
+        try:
+            pCounterValue
+        except NameError:
+            main.log.error( "pCounterValue not defined, setting to 0" )
+            pCounterValue = 0
+        try:
+            iCounterValue
+        except NameError:
+            main.log.error( "iCounterValue not defined, setting to 0" )
+            iCounterValue = 0
+        try:
+            onosSet
+        except NameError:
+            main.log.error( "onosSet not defined, setting to empty Set" )
+            onosSet = set([])
+        # Variables for the distributed primitives tests. These are local only
+        addValue = "a"
+        addAllValue = "a b c d e f"
+        retainValue = "c d e f"
+
+        description = "Check for basic functionality with distributed " +\
+                      "primitives"
+        main.case( description )
+        main.caseExplaination = "Test the methods of the distributed primitives (counters and sets) throught the cli"
+        # DISTRIBUTED ATOMIC COUNTERS
+        main.step( "Increment and get a default counter on each node" )
+        pCounters = []
+        threads = []
+        addedPValues = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counterTestIncrement,
+                             name="counterIncrement-" + str( i ),
+                             args=[ pCounterName ] )
+            pCounterValue += 1
+            addedPValues.append( pCounterValue )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            pCounters.append( t.result )
+        # Check that counter incremented numController times
+        pCounterResults = True
+        for i in addedPValues:
+            tmpResult = i in pCounters
+            pCounterResults = pCounterResults and tmpResult
+            if not tmpResult:
+                main.log.error( str( i ) + " is not in partitioned "
+                                "counter incremented results" )
+        utilities.assert_equals( expect=True,
+                                 actual=pCounterResults,
+                                 onpass="Default counter incremented",
+                                 onfail="Error incrementing default" +
+                                        " counter" )
+
+        main.step( "Increment and get an in memory counter on each node" )
+        iCounters = []
+        addedIValues = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counterTestIncrement,
+                             name="icounterIncrement-" + str( i ),
+                             args=[ iCounterName ],
+                             kwargs={ "inMemory": True } )
+            iCounterValue += 1
+            addedIValues.append( iCounterValue )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            iCounters.append( t.result )
+        # Check that counter incremented numController times
+        iCounterResults = True
+        for i in addedIValues:
+            tmpResult = i in iCounters
+            iCounterResults = iCounterResults and tmpResult
+            if not tmpResult:
+                main.log.error( str( i ) + " is not in the in-memory "
+                                "counter incremented results" )
+        utilities.assert_equals( expect=True,
+                                 actual=iCounterResults,
+                                 onpass="In memory counter incremented",
+                                 onfail="Error incrementing in memory" +
+                                        " counter" )
+
+        main.step( "Check counters are consistant across nodes" )
+        onosCounters = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counters,
+                             name="counters-" + str( i ) )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            onosCounters.append( t.result )
+        tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+        if all( tmp ):
+            main.log.info( "Counters are consistent across all nodes" )
+            consistentCounterResults = main.TRUE
+        else:
+            main.log.error( "Counters are not consistent across all nodes" )
+            consistentCounterResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=consistentCounterResults,
+                                 onpass="ONOS counters are consistent " +
+                                        "across nodes",
+                                 onfail="ONOS Counters are inconsistent " +
+                                        "across nodes" )
+
+        main.step( "Counters we added have the correct values" )
+        correctResults = main.TRUE
+        for i in range( numControllers ):
+            current = json.loads( onosCounters[i] )
+            pValue = None
+            iValue = None
+            try:
+                for database in current:
+                    partitioned = database.get( 'partitionedDatabaseCounters' )
+                    if partitioned:
+                        for value in partitioned:
+                            if value.get( 'name' ) == pCounterName:
+                                pValue = value.get( 'value' )
+                                break
+                    inMemory = database.get( 'inMemoryDatabaseCounters' )
+                    if inMemory:
+                        for value in inMemory:
+                            if value.get( 'name' ) == iCounterName:
+                                iValue = value.get( 'value' )
+                                break
+            except AttributeError, e:
+                main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
+                                "is not as expected" )
+                correctResults = main.FALSE
+            if pValue == pCounterValue:
+                main.log.info( "Partitioned counter value is correct" )
+            else:
+                main.log.error( "Partitioned counter value is incorrect," +
+                                " expected value: " + str( pCounterValue )
+                                + " current value: " + str( pValue ) )
+                correctResults = main.FALSE
+            if iValue == iCounterValue:
+                main.log.info( "In memory counter value is correct" )
+            else:
+                main.log.error( "In memory counter value is incorrect, " +
+                                "expected value: " + str( iCounterValue ) +
+                                " current value: " + str( iValue ) )
+                correctResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=correctResults,
+                                 onpass="Added counters are correct",
+                                 onfail="Added counters are incorrect" )
+        # DISTRIBUTED SETS
+        main.step( "Distributed Set get" )
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=getResults,
+                                 onpass="Set elements are correct",
+                                 onfail="Set elements are incorrect" )
+
+        main.step( "Distributed Set size" )
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=sizeResults,
+                                 onpass="Set sizes are correct",
+                                 onfail="Set sizes are incorrect" )
+
+        main.step( "Distributed Set add()" )
+        onosSet.add( addValue )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAdd-" + str( i ),
+                             args=[ onosSetName, addValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addResults = main.FALSE
+            else:
+                # unexpected result
+                addResults = main.FALSE
+        if addResults != main.TRUE:
+            main.log.error( "Error executing set add" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addResults = addResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addResults,
+                                 onpass="Set add correct",
+                                 onfail="Set add was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set contains()" )
+        containsResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setContains-" + str( i ),
+                             args=[ onosSetName ],
+                             kwargs={ "values": addValue } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            # NOTE: This is the tuple
+            containsResponses.append( t.result )
+
+        containsResults = main.TRUE
+        for i in range( numControllers ):
+            if containsResponses[ i ] == main.ERROR:
+                containsResults = main.FALSE
+            else:
+                containsResults = containsResults and\
+                                  containsResponses[ i ][ 1 ]
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=containsResults,
+                                 onpass="Set contains is functional",
+                                 onfail="Set contains failed" )
+
+        main.step( "Distributed Set containsAll()" )
+        containsAllResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setContainsAll-" + str( i ),
+                             args=[ onosSetName ],
+                             kwargs={ "values": addAllValue } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            # NOTE: This is the tuple
+            containsAllResponses.append( t.result )
+
+        containsAllResults = main.TRUE
+        for i in range( numControllers ):
+            if containsResponses[ i ] == main.ERROR:
+                containsResults = main.FALSE
+            else:
+                containsResults = containsResults and\
+                                  containsResponses[ i ][ 1 ]
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=containsAllResults,
+                                 onpass="Set containsAll is functional",
+                                 onfail="Set containsAll failed" )
+
+        main.step( "Distributed Set remove()" )
+        onosSet.remove( addValue )
+        removeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestRemove-" + str( i ),
+                             args=[ onosSetName, addValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            removeResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        removeResults = main.TRUE
+        for i in range( numControllers ):
+            if removeResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif removeResponses[ i ] == main.FALSE:
+                # not in set, probably fine
+                pass
+            elif removeResponses[ i ] == main.ERROR:
+                # Error in execution
+                removeResults = main.FALSE
+            else:
+                # unexpected result
+                removeResults = main.FALSE
+        if removeResults != main.TRUE:
+            main.log.error( "Error executing set remove" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        removeResults = removeResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=removeResults,
+                                 onpass="Set remove correct",
+                                 onfail="Set remove was incorrect" )
+
+        main.step( "Distributed Set removeAll()" )
+        onosSet.difference_update( addAllValue.split() )
+        removeAllResponses = []
+        threads = []
+        try:
+            for i in range( numControllers ):
+                t = main.Thread( target=CLIs[i].setTestRemove,
+                                 name="setTestRemoveAll-" + str( i ),
+                                 args=[ onosSetName, addAllValue ] )
+                threads.append( t )
+                t.start()
+            for t in threads:
+                t.join()
+                removeAllResponses.append( t.result )
+        except Exception, e:
+            main.log.exception(e)
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        removeAllResults = main.TRUE
+        for i in range( numControllers ):
+            if removeAllResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif removeAllResponses[ i ] == main.FALSE:
+                # not in set, probably fine
+                pass
+            elif removeAllResponses[ i ] == main.ERROR:
+                # Error in execution
+                removeAllResults = main.FALSE
+            else:
+                # unexpected result
+                removeAllResults = main.FALSE
+        if removeAllResults != main.TRUE:
+            main.log.error( "Error executing set removeAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        removeAllResults = removeAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=removeAllResults,
+                                 onpass="Set removeAll correct",
+                                 onfail="Set removeAll was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set clear()" )
+        onosSet.clear()
+        clearResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestClear-" + str( i ),
+                             args=[ onosSetName, " "],  # Values doesn't matter
+                             kwargs={ "clear": True } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            clearResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        clearResults = main.TRUE
+        for i in range( numControllers ):
+            if clearResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif clearResponses[ i ] == main.FALSE:
+                # Nothing set, probably fine
+                pass
+            elif clearResponses[ i ] == main.ERROR:
+                # Error in execution
+                clearResults = main.FALSE
+            else:
+                # unexpected result
+                clearResults = main.FALSE
+        if clearResults != main.TRUE:
+            main.log.error( "Error executing set clear" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        clearResults = clearResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=clearResults,
+                                 onpass="Set clear correct",
+                                 onfail="Set clear was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set retain()" )
+        onosSet.intersection_update( retainValue.split() )
+        retainResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestRetain-" + str( i ),
+                             args=[ onosSetName, retainValue ],
+                             kwargs={ "retain": True } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            retainResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        retainResults = main.TRUE
+        for i in range( numControllers ):
+            if retainResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif retainResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif retainResponses[ i ] == main.ERROR:
+                # Error in execution
+                retainResults = main.FALSE
+            else:
+                # unexpected result
+                retainResults = main.FALSE
+        if retainResults != main.TRUE:
+            main.log.error( "Error executing set retain" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " +
+                                str( size ) + " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        retainResults = retainResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=retainResults,
+                                 onpass="Set retain correct",
+                                 onfail="Set retain was incorrect" )
+
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index 595b78c..956d889 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -1,11 +1,12 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
         <cellName>HA</cellName>
     </ENV>
     <Git>False</Git>
     <branch> master </branch>
     <num_controllers> 7 </num_controllers>
+    <tcpdump> False </tcpdump>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 43dd995..af96bb5 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -18,6 +18,8 @@
 CASE13: Clean up
 CASE14: start election app on all onos nodes
 CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
 """
 
 
@@ -44,9 +46,12 @@
         start cli sessions
         start tcpdump
         """
-        main.log.report(
-            "ONOS HA test: Restart minority of ONOS nodes - initialization" )
+        main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " +
+                         "initialization" )
         main.case( "Setting up test environment" )
+        main.caseExplaination = "Setup the test environment including " +\
+                                "installing ONOS, starting Mininet and ONOS" +\
+                                "cli sessions."
         # TODO: save all the timers and output them for plotting
 
         # load some variables from the params file
@@ -90,36 +95,73 @@
         verifyResult = main.ONOSbench.verifyCell()
 
         # FIXME:this is short term fix
-        main.log.report( "Removing raft logs" )
+        main.log.info( "Removing raft logs" )
         main.ONOSbench.onosRemoveRaftLogs()
 
-        main.log.report( "Uninstalling ONOS" )
+        main.log.info( "Uninstalling ONOS" )
         for node in nodes:
             main.ONOSbench.onosUninstall( node.ip_address )
 
+        # Make sure ONOS is DEAD
+        main.log.info( "Killing any ONOS processes" )
+        killResults = main.TRUE
+        for node in nodes:
+            killed = main.ONOSbench.onosKill( node.ip_address )
+            killResults = killResults and killed
+
         cleanInstallResult = main.TRUE
         gitPullResult = main.TRUE
 
         main.step( "Starting Mininet" )
-        main.Mininet1.startNet( )
+        mnResult = main.Mininet1.startNet( )
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet Started",
+                                 onfail="Error starting Mininet" )
 
-        main.step( "Compiling the latest version of ONOS" )
+        main.step( "Git checkout and pull " + gitBranch )
         if PULLCODE:
-            main.step( "Git checkout and pull " + gitBranch )
             main.ONOSbench.gitCheckout( gitBranch )
             gitPullResult = main.ONOSbench.gitPull()
-            if gitPullResult == main.ERROR:
-                main.log.error( "Error pulling git branch" )
+            # values of 1 or 3 are good
+            utilities.assert_lesser( expect=0, actual=gitPullResult,
+                                      onpass="Git pull successful",
+                                      onfail="Git pull failed" )
+        main.ONOSbench.getVersion( report=True )
 
-            main.step( "Using mvn clean & install" )
+        main.step( "Using mvn clean install" )
+        cleanInstallResult = main.TRUE
+        if PULLCODE and gitPullResult == main.TRUE:
             cleanInstallResult = main.ONOSbench.cleanInstall()
         else:
             main.log.warn( "Did not pull new code so skipping mvn " +
                            "clean install" )
-        main.ONOSbench.getVersion( report=True )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=cleanInstallResult,
+                                 onpass="MCI successful",
+                                 onfail="MCI failed" )
+        # GRAPHS
+        # NOTE: important params here:
+        #       job = name of Jenkins job
+        #       Plot Name = Plot-HA, only can be used if multiple plots
+        #       index = The number of the graph under plot name
+        job = "HAMinorityRestart"
+        plotName = "Plot-HA"
+        graphs = '<ac:structured-macro ac:name="html">\n'
+        graphs += '<ac:plain-text-body><![CDATA[\n'
+        graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+                  '/plot/' + plotName + '/getPlot?index=0' +\
+                  '&width=500&height=300"' +\
+                  'noborder="0" width="500" height="300" scrolling="yes" ' +\
+                  'seamless="seamless"></iframe>\n'
+        graphs += ']]></ac:plain-text-body>\n'
+        graphs += '</ac:structured-macro>\n'
+        main.log.wiki(graphs)
 
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()
+        utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+                                 onpass="ONOS package successful",
+                                 onfail="ONOS package failed" )
 
         main.step( "Installing ONOS package" )
         onosInstallResult = main.TRUE
@@ -127,6 +169,9 @@
             tmpResult = main.ONOSbench.onosInstall( options="-f",
                                                     node=node.ip_address )
             onosInstallResult = onosInstallResult and tmpResult
+        utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+                                 onpass="ONOS install successful",
+                                 onfail="ONOS install failed" )
 
         main.step( "Checking if ONOS is up yet" )
         for i in range( 2 ):
@@ -134,12 +179,15 @@
             for node in nodes:
                 started = main.ONOSbench.isup( node.ip_address )
                 if not started:
-                    main.log.report( node.name + " didn't start!" )
+                    main.log.error( node.name + " didn't start!" )
                     main.ONOSbench.onosStop( node.ip_address )
                     main.ONOSbench.onosStart( node.ip_address )
                 onosIsupResult = onosIsupResult and started
             if onosIsupResult == main.TRUE:
                 break
+        utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+                                 onpass="ONOS startup successful",
+                                 onfail="ONOS startup failed" )
 
         main.log.step( "Starting ONOS CLI sessions" )
         cliResults = main.TRUE
@@ -154,14 +202,19 @@
         for t in threads:
             t.join()
             cliResults = cliResults and t.result
+        utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                 onpass="ONOS cli startup successful",
+                                 onfail="ONOS cli startup failed" )
 
-        main.step( "Start Packet Capture MN" )
-        main.Mininet2.startTcpdump(
-            str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
-            + "-MN.pcap",
-            intf=main.params[ 'MNtcpdump' ][ 'intf' ],
-            port=main.params[ 'MNtcpdump' ][ 'port' ] )
+        if main.params[ 'tcpdump' ].lower() == "true":
+            main.step( "Start Packet Capture MN" )
+            main.Mininet2.startTcpdump(
+                str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+                + "-MN.pcap",
+                intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+                port=main.params[ 'MNtcpdump' ][ 'port' ] )
 
+        main.step( "App Ids check" )
         appCheck = main.TRUE
         threads = []
         for i in range( numControllers ):
@@ -174,22 +227,15 @@
         for t in threads:
             t.join()
             appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
         if appCheck != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
-        case1Result = ( cleanInstallResult and packageResult and
-                        cellResult and verifyResult and onosInstallResult
-                        and onosIsupResult and cliResults and appCheck)
-
-        utilities.assert_equals( expect=main.TRUE, actual=case1Result,
-                                 onpass="Test startup successful",
-                                 onfail="Test startup NOT successful" )
-
-        if case1Result == main.FALSE:
+        if cliResults == main.FALSE:
+            main.log.error( "Failed to start ONOS, stopping test" )
             main.cleanup()
             main.exit()
 
@@ -198,6 +244,7 @@
         Assign mastership to controllers
         """
         import re
+        import time
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -211,8 +258,12 @@
         assert ONOS6Port, "ONOS6Port not defined"
         assert ONOS7Port, "ONOS7Port not defined"
 
-        main.log.report( "Assigning switches to controllers" )
         main.case( "Assigning Controllers" )
+        main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+                                "and check that an ONOS node becomes the " +\
+                                "master of the device. Then manually assign" +\
+                                " mastership to specific ONOS nodes using" +\
+                                " 'device-role'"
         main.step( "Assign switches to controllers" )
 
         # TODO: rewrite this function to take lists of ips and ports?
@@ -244,16 +295,18 @@
                                     "not in the list of controllers s" +
                                     str( i ) + " is connecting to." )
                     mastershipCheck = main.FALSE
-        if mastershipCheck == main.TRUE:
-            main.log.report( "Switch mastership assigned correctly" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=mastershipCheck,
             onpass="Switch mastership assigned correctly",
             onfail="Switches not assigned correctly to controllers" )
+
+        main.step( "Assign mastership of switches to specific controllers" )
         # Manually assign mastership to the controller we want
         roleCall = main.TRUE
-        roleCheck = main.TRUE
+
+        ipList = [ ]
+        deviceList = []
         try:
             for i in range( 1, 29 ):  # switches 1 through 28
                 # set up correct variables:
@@ -297,16 +350,8 @@
                 # TODO: make this controller dynamic
                 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
                                                                   ip )
-                # Check assignment
-                master =  main.ONOScli1.getRole( deviceId ).get( 'master' )
-                if ip in master:
-                    roleCheck = roleCheck and main.TRUE
-                else:
-                    roleCheck = roleCheck and main.FALSE
-                    main.log.error( "Error, controller " + ip + " is not" +
-                                    " master " + "of device " +
-                                    str( deviceId ) + ". Master is " +
-                                    repr( master ) + "." )
+                ipList.append( ip )
+                deviceList.append( deviceId )
         except ( AttributeError, AssertionError ):
             main.log.exception( "Something is wrong with ONOS device view" )
             main.log.info( main.ONOScli1.devices() )
@@ -316,6 +361,24 @@
             onpass="Re-assigned switch mastership to designated controller",
             onfail="Something wrong with deviceRole calls" )
 
+        main.step( "Check mastership was correctly assigned" )
+        roleCheck = main.TRUE
+        # NOTE: This is due to the fact that device mastership change is not
+        #       atomic and is actually a multi step process
+        time.sleep( 5 )
+        for i in range( len( ipList ) ):
+            ip = ipList[i]
+            deviceId = deviceList[i]
+            # Check assignment
+            master = main.ONOScli1.getRole( deviceId ).get( 'master' )
+            if ip in master:
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+                main.log.error( "Error, controller " + ip + " is not" +
+                                " master " + "of device " +
+                                str( deviceId ) + ". Master is " +
+                                repr( master ) + "." )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=roleCheck,
@@ -339,17 +402,21 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        main.log.report( "Adding host intents" )
         main.case( "Adding host Intents" )
-
-        main.step( "Discovering  Hosts( Via pingall for now )" )
-        # FIXME: Once we have a host discovery mechanism, use that instead
+        main.caseExplaination = "Discover hosts by using pingall then " +\
+                                "assign predetermined host-to-host intents." +\
+                                " After installation, check that the intent" +\
+                                " is distributed to all nodes and the state" +\
+                                " is INSTALLED"
 
         # install onos-app-fwd
-        main.log.info( "Install reactive forwarding app" )
-        appResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+        main.step( "Install reactive forwarding app" )
+        installResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+        utilities.assert_equals( expect=main.TRUE, actual=installResults,
+                                 onpass="Install fwd successful",
+                                 onfail="Install fwd failed" )
 
-        # FIXME: add this to asserts
+        main.step( "Check app ids" )
         appCheck = main.TRUE
         threads = []
         for i in range( numControllers ):
@@ -362,13 +429,15 @@
         for t in threads:
             t.join()
             appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
         if appCheck != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
+        main.step( "Discovering Hosts( Via pingall for now )" )
+        # FIXME: Once we have a host discovery mechanism, use that instead
         # REACTIVE FWD test
         pingResult = main.FALSE
         for i in range(2):  # Retry if pingall fails first time
@@ -378,14 +447,23 @@
                 expect=main.TRUE,
                 actual=pingResult,
                 onpass="Reactive Pingall test passed",
-                onfail="Reactive Pingall failed, one or more ping pairs failed" )
+                onfail="Reactive Pingall failed, " +
+                       "one or more ping pairs failed" )
             time2 = time.time()
-            main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
-
+            main.log.info( "Time for pingall: %2f seconds" %
+                           ( time2 - time1 ) )
+        # timeout for fwd flows
+        time.sleep( 11 )
         # uninstall onos-app-fwd
-        main.log.info( "Uninstall reactive forwarding app" )
-        appResults = appResults and CLIs[0].deactivateApp( "org.onosproject.fwd" )
+        main.step( "Uninstall reactive forwarding app" )
+        uninstallResult = CLIs[0].deactivateApp( "org.onosproject.fwd" )
+        utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
+                                 onpass="Uninstall fwd successful",
+                                 onfail="Uninstall fwd failed" )
+
+        main.step( "Check app ids" )
         threads = []
+        appCheck2 = main.TRUE
         for i in range( numControllers ):
             t = main.Thread( target=CLIs[i].appToIDCheck,
                              name="appToIDCheck-" + str( i ),
@@ -395,18 +473,15 @@
 
         for t in threads:
             t.join()
-            appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
-        if appCheck != main.TRUE:
+            appCheck2 = appCheck2 and t.result
+        if appCheck2 != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
-        # timeout for fwd flows
-        time.sleep( 11 )
-
-        main.step( "Add host intents" )
+        main.step( "Add host intents via cli" )
         intentIds = []
         # TODO:  move the host numbers to params
         #        Maybe look at all the paths we ping?
@@ -449,7 +524,10 @@
                 except ( ValueError, TypeError ):
                     main.log.warn( repr( hosts ) )
                 hostResult = main.FALSE
-        # FIXME: DEBUG
+        utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+                                 onpass="Found a host id for each host",
+                                 onfail="Error looking up host ids" )
+
         intentStart = time.time()
         onosIds = main.ONOScli1.getAllIntentsId()
         main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -544,35 +622,48 @@
             main.log.exception( "Error parsing pending map" )
             main.log.error( repr( pendingMap ) )
 
-        intentAddResult = bool( pingResult and hostResult and intentAddResult
-                                and not missingIntents and installedCheck )
-        utilities.assert_equals(
-            expect=True,
-            actual=intentAddResult,
-            onpass="Pushed host intents to ONOS",
-            onfail="Error in pushing host intents to ONOS" )
+        intentAddResult = bool( intentAddResult and not missingIntents and
+                                installedCheck )
+        if not intentAddResult:
+            main.log.error( "Error in pushing host intents to ONOS" )
+
+        main.step( "Intent Anti-Entropy dispersion" )
         for i in range(100):
-            onosIds = main.ONOScli1.getAllIntentsId()
+            correct = True
             main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
-            main.log.info( "Intents in ONOS: " + str( sorted( onosIds ) ) )
-            if sorted(onosIds) == sorted(intentIds):
+            for cli in CLIs:
+                onosIds = []
+                ids = cli.getAllIntentsId()
+                onosIds.append( ids )
+                main.log.debug( "Intents in " + cli.name + ": " +
+                                str( sorted( onosIds ) ) )
+                if sorted( ids ) != sorted( intentIds ):
+                    correct = False
+                    break
+                else:
+                    intents = json.loads( cli.intents() )
+                    for intent in intents:
+                        if intent[ 'state' ] != "INSTALLED":
+                            correct = False
+                            break
+            if correct:
                 break
             else:
                 time.sleep(1)
-        # FIXME: DEBUG
         if not intentStop:
             intentStop = time.time()
+        global gossipTime
         gossipTime = intentStop - intentStart
         main.log.info( "It took about " + str( gossipTime ) +
-                        " seconds for all intents to appear on ONOS1" )
+                        " seconds for all intents to appear in each node" )
         # FIXME: make this time configurable/calculate based off of number of
         #        nodes and gossip rounds
         utilities.assert_greater_equals(
-                expect=30, actual=gossipTime,
+                expect=40, actual=gossipTime,
                 onpass="ECM anti-entropy for intents worked within " +
                        "expected time",
                 onfail="Intent ECM anti-entropy took too long" )
-        if gossipTime <= 30:
+        if gossipTime <= 40:
             intentAddResult = True
 
         if not intentAddResult or "key" in pendingMap:
@@ -675,9 +766,11 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        description = " Ping across added host intents"
-        main.log.report( description )
-        main.case( description )
+        main.case( "Verify connectivity by sendind traffic across Intents" )
+        main.caseExplaination = "Ping across added host intents to check " +\
+                                "functionality and check the state of " +\
+                                "the intent"
+        main.step( "Ping across added host intents" )
         PingResult = main.TRUE
         for i in range( 8, 18 ):
             ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -690,7 +783,7 @@
                 main.log.info( "Ping test passed!" )
                 # Don't set PingResult or you'd override failures
         if PingResult == main.FALSE:
-            main.log.report(
+            main.log.error(
                 "Intents have not been installed correctly, pings failed." )
             # TODO: pretty print
             main.log.warn( "ONOS1 intents: " )
@@ -702,17 +795,17 @@
                                            separators=( ',', ': ' ) ) )
             except ( ValueError, TypeError ):
                 main.log.warn( repr( tmpIntents ) )
-        if PingResult == main.TRUE:
-            main.log.report(
-                "Intents have been installed correctly and verified by pings" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=PingResult,
             onpass="Intents have been installed correctly and pings work",
             onfail="Intents have not been installed correctly, pings failed." )
 
-        installedCheck = True
-        if PingResult is not main.TRUE:
+        main.step( "Check Intent state" )
+        installedCheck = False
+        count = 0
+        while not installedCheck and count < 40:
+            installedCheck = True
             # Print the intent states
             intents = main.ONOScli1.intents()
             intentStates = []
@@ -728,66 +821,88 @@
                     intentStates.append( ( intentId, state ) )
             except ( ValueError, TypeError ):
                 main.log.exception( "Error parsing intents." )
+            # Print states
             intentStates.sort()
             for i, s in intentStates:
                 count += 1
                 main.log.info( "%-6s%-15s%-15s" %
                                ( str( count ), str( i ), str( s ) ) )
-            leaders = main.ONOScli1.leaders()
-            try:
-                if leaders:
-                    parsedLeaders = json.loads( leaders )
-                    main.log.warn( json.dumps( parsedLeaders,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # check for all intent partitions
-                    # check for election
-                    topics = []
-                    for i in range( 14 ):
-                        topics.append( "intent-partition-" + str( i ) )
-                    # FIXME: this should only be after we start the app
-                    topics.append( "org.onosproject.election" )
-                    main.log.debug( topics )
-                    ONOStopics = [ j['topic'] for j in parsedLeaders ]
-                    for topic in topics:
-                        if topic not in ONOStopics:
-                            main.log.error( "Error: " + topic +
-                                            " not in leaders" )
-                else:
-                    main.log.error( "leaders() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing leaders" )
-                main.log.error( repr( leaders ) )
-            partitions = main.ONOScli1.partitions()
-            try:
-                if partitions :
-                    parsedPartitions = json.loads( partitions )
-                    main.log.warn( json.dumps( parsedPartitions,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # TODO check for a leader in all paritions
-                    # TODO check for consistency among nodes
-                else:
-                    main.log.error( "partitions() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing partitions" )
-                main.log.error( repr( partitions ) )
-            pendingMap = main.ONOScli1.pendingMap()
-            try:
-                if pendingMap :
-                    parsedPending = json.loads( pendingMap )
-                    main.log.warn( json.dumps( parsedPending,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # TODO check something here?
-                else:
-                    main.log.error( "pendingMap() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing pending map" )
-                main.log.error( repr( pendingMap ) )
+            if not installedCheck:
+                time.sleep( 1 )
+                count += 1
+        utilities.assert_equals( expect=True, actual=installedCheck,
+                                 onpass="Intents are all INSTALLED",
+                                 onfail="Intents are not all in " +
+                                        "INSTALLED state" )
+
+        main.step( "Check leadership of topics" )
+        leaders = main.ONOScli1.leaders()
+        topicCheck = main.TRUE
+        try:
+            if leaders:
+                parsedLeaders = json.loads( leaders )
+                main.log.warn( json.dumps( parsedLeaders,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # check for all intent partitions
+                # check for election
+                # TODO: Look at Devices as topics now that it uses this system
+                topics = []
+                for i in range( 14 ):
+                    topics.append( "intent-partition-" + str( i ) )
+                # FIXME: this should only be after we start the app
+                # FIXME: topics.append( "org.onosproject.election" )
+                # Print leaders output
+                main.log.debug( topics )
+                ONOStopics = [ j['topic'] for j in parsedLeaders ]
+                for topic in topics:
+                    if topic not in ONOStopics:
+                        main.log.error( "Error: " + topic +
+                                        " not in leaders" )
+                        topicCheck = main.FALSE
+            else:
+                main.log.error( "leaders() returned None" )
+                topicCheck = main.FALSE
+        except ( ValueError, TypeError ):
+            topicCheck = main.FALSE
+            main.log.exception( "Error parsing leaders" )
+            main.log.error( repr( leaders ) )
+            # TODO: Check for a leader of these topics
+        utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+                                 onpass="intent Partitions is in leaders",
+                                 onfail="Some topics were lost " )
+        # Print partitions
+        partitions = main.ONOScli1.partitions()
+        try:
+            if partitions :
+                parsedPartitions = json.loads( partitions )
+                main.log.warn( json.dumps( parsedPartitions,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # TODO check for a leader in all paritions
+                # TODO check for consistency among nodes
+            else:
+                main.log.error( "partitions() returned None" )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing partitions" )
+            main.log.error( repr( partitions ) )
+        # Print Pending Map
+        pendingMap = main.ONOScli1.pendingMap()
+        try:
+            if pendingMap :
+                parsedPending = json.loads( pendingMap )
+                main.log.warn( json.dumps( parsedPending,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # TODO check something here?
+            else:
+                main.log.error( "pendingMap() returned None" )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing pending map" )
+            main.log.error( repr( pendingMap ) )
 
         if not installedCheck:
             main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -868,6 +983,39 @@
             except ( ValueError, TypeError ):
                 main.log.exception( "Error parsing pending map" )
                 main.log.error( repr( pendingMap ) )
+        # Print flowrules
+        main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+        main.step( "Wait a minute then ping again" )
+        # the wait is above
+        PingResult = main.TRUE
+        for i in range( 8, 18 ):
+            ping = main.Mininet1.pingHost( src="h" + str( i ),
+                                           target="h" + str( i + 10 ) )
+            PingResult = PingResult and ping
+            if ping == main.FALSE:
+                main.log.warn( "Ping failed between h" + str( i ) +
+                               " and h" + str( i + 10 ) )
+            elif ping == main.TRUE:
+                main.log.info( "Ping test passed!" )
+                # Don't set PingResult or you'd override failures
+        if PingResult == main.FALSE:
+            main.log.error(
+                "Intents have not been installed correctly, pings failed." )
+            # TODO: pretty print
+            main.log.warn( "ONOS1 intents: " )
+            try:
+                tmpIntents = main.ONOScli1.intents()
+                main.log.warn( json.dumps( json.loads( tmpIntents ),
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+            except ( ValueError, TypeError ):
+                main.log.warn( repr( tmpIntents ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=PingResult,
+            onpass="Intents have been installed correctly and pings work",
+            onfail="Intents have not been installed correctly, pings failed." )
 
     def CASE5( self, main ):
         """
@@ -883,7 +1031,6 @@
         # assumes that sts is already in you PYTHONPATH
         from sts.topology.teston_topology import TestONTopology
 
-        main.log.report( "Setting up and gathering data for current state" )
         main.case( "Setting up and gathering data for current state" )
         # The general idea for this test case is to pull the state of
         # ( intents,flows, topology,... ) from each ONOS node
@@ -931,7 +1078,7 @@
 
         for i in range( numControllers ):
             if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " roles" )
                 main.log.warn(
                     "ONOS" + str( i + 1 ) + " mastership response: " +
@@ -945,7 +1092,7 @@
 
         main.step( "Check for consistency in roles from each controller" )
         if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
-            main.log.report(
+            main.log.info(
                 "Switch roles are consistent across all ONOS nodes" )
         else:
             consistentMastership = False
@@ -993,7 +1140,7 @@
 
         for i in range( numControllers ):
             if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " intents" )
                 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
                                repr( ONOSIntents[ i ] ) )
@@ -1006,34 +1153,63 @@
 
         main.step( "Check for consistency in Intents from each controller" )
         if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
-            main.log.report( "Intents are consistent across all ONOS " +
+            main.log.info( "Intents are consistent across all ONOS " +
                              "nodes" )
         else:
             consistentIntents = False
-            main.log.report( "Intents not consistent" )
+            main.log.error( "Intents not consistent" )
         utilities.assert_equals(
             expect=True,
             actual=consistentIntents,
             onpass="Intents are consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of intents" )
 
+        if intentsResults:
+            # Try to make it easy to figure out what is happening
+            #
+            # Intent      ONOS1      ONOS2    ...
+            #  0x01     INSTALLED  INSTALLING
+            #  ...        ...         ...
+            #  ...        ...         ...
+            title = "   Id"
+            for n in range( numControllers ):
+                title += " " * 10 + "ONOS" + str( n + 1 )
+            main.log.warn( title )
+            # get all intent keys in the cluster
+            keys = []
+            for nodeStr in ONOSIntents:
+                node = json.loads( nodeStr )
+                for intent in node:
+                    keys.append( intent.get( 'id' ) )
+            keys = set( keys )
+            for key in keys:
+                row = "%-13s" % key
+                for nodeStr in ONOSIntents:
+                    node = json.loads( nodeStr )
+                    for intent in node:
+                        if intent.get( 'id', "Error" ) == key:
+                            row += "%-15s" % intent.get( 'state' )
+                main.log.warn( row )
+            # End table view
+
         if intentsResults and not consistentIntents:
+            # print the json objects
             n = len(ONOSIntents)
-            main.log.warn( "ONOS" + str( n ) + " intents: " )
-            main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
-                                       sort_keys=True,
-                                       indent=4,
-                                       separators=( ',', ': ' ) ) )
+            main.log.debug( "ONOS" + str( n ) + " intents: " )
+            main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+                                        sort_keys=True,
+                                        indent=4,
+                                        separators=( ',', ': ' ) ) )
             for i in range( numControllers ):
                 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
-                    main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
-                    main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
+                    main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
+                    main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
+                                                sort_keys=True,
+                                                indent=4,
+                                                separators=( ',', ': ' ) ) )
                 else:
-                    main.log.warn( nodes[ i ].name + " intents match ONOS" +
-                                   str( n ) + " intents" )
+                    main.log.debug( nodes[ i ].name + " intents match ONOS" +
+                                    str( n ) + " intents" )
         elif intentsResults and consistentIntents:
             intentCheck = main.TRUE
             intentState = ONOSIntents[ 0 ]
@@ -1065,7 +1241,7 @@
         for i in range( numControllers ):
             num = str( i + 1 )
             if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
-                main.log.report( "Error in getting ONOS" + num + " flows" )
+                main.log.error( "Error in getting ONOS" + num + " flows" )
                 main.log.warn( "ONOS" + num + " flows response: " +
                                repr( ONOSFlows[ i ] ) )
                 flowsResults = False
@@ -1089,7 +1265,7 @@
         main.step( "Check for consistency in Flows from each controller" )
         tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
         if all( tmp ):
-            main.log.report( "Flow count is consistent across all ONOS nodes" )
+            main.log.info( "Flow count is consistent across all ONOS nodes" )
         else:
             consistentFlows = False
         utilities.assert_equals(
@@ -1244,6 +1420,7 @@
         # Compare json objects for hosts and dataplane clusters
 
         # hosts
+        main.step( "Host view is consistent across ONOS nodes" )
         consistentHostsResult = main.TRUE
         for controller in range( len( hosts ) ):
             controllerStr = str( controller + 1 )
@@ -1251,14 +1428,14 @@
                 if hosts[ controller ] == hosts[ 0 ]:
                     continue
                 else:  # hosts not consistent
-                    main.log.report( "hosts from ONOS" +
+                    main.log.error( "hosts from ONOS" +
                                      controllerStr +
                                      " is inconsistent with ONOS1" )
                     main.log.warn( repr( hosts[ controller ] ) )
                     consistentHostsResult = main.FALSE
 
             else:
-                main.log.report( "Error in getting ONOS hosts from ONOS" +
+                main.log.error( "Error in getting ONOS hosts from ONOS" +
                                  controllerStr )
                 consistentHostsResult = main.FALSE
                 main.log.warn( "ONOS" + controllerStr +
@@ -1270,11 +1447,12 @@
             onpass="Hosts view is consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of hosts" )
 
+        main.step( "Each host has an IP address" )
         ipResult = main.TRUE
         for controller in range( 0, len( hosts ) ):
             controllerStr = str( controller + 1 )
             for host in hosts[ controller ]:
-                if not host.get( 'ips', [ ] ):
+                if not host.get( 'ipAddresses', [ ] ):
                     main.log.error( "DEBUG:Error with host ips on controller" +
                                     controllerStr + ": " + str( host ) )
                     ipResult = main.FALSE
@@ -1285,6 +1463,7 @@
             onfail="The ip of at least one host is missing" )
 
         # Strongly connected clusters of devices
+        main.step( "Cluster view is consistent across ONOS nodes" )
         consistentClustersResult = main.TRUE
         for controller in range( len( clusters ) ):
             controllerStr = str( controller + 1 )
@@ -1292,12 +1471,12 @@
                 if clusters[ controller ] == clusters[ 0 ]:
                     continue
                 else:  # clusters not consistent
-                    main.log.report( "clusters from ONOS" + controllerStr +
+                    main.log.error( "clusters from ONOS" + controllerStr +
                                      " is inconsistent with ONOS1" )
                     consistentClustersResult = main.FALSE
 
             else:
-                main.log.report( "Error in getting dataplane clusters " +
+                main.log.error( "Error in getting dataplane clusters " +
                                  "from ONOS" + controllerStr )
                 consistentClustersResult = main.FALSE
                 main.log.warn( "ONOS" + controllerStr +
@@ -1309,6 +1488,7 @@
             onpass="Clusters view is consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of clusters" )
         # there should always only be one cluster
+        main.step( "Cluster view correct across ONOS nodes" )
         try:
             numClusters = len( json.loads( clusters[ 0 ] ) )
         except ( ValueError, TypeError ):
@@ -1379,14 +1559,6 @@
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
 
-        finalAssert = main.TRUE
-        finalAssert = ( finalAssert and topoResult and flowCheck
-                        and intentCheck and consistentMastership
-                        and mastershipCheck and rolesNotNull )
-        utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
-                                 onpass="State check successful",
-                                 onfail="State check NOT successful" )
-
     def CASE6( self, main ):
         """
         The Failure case.
@@ -1397,15 +1569,21 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        main.log.report( "Killing 3 ONOS nodes" )
         main.case( "Restart minority of ONOS nodes" )
-        # TODO: Randomize these nodes
+        main.step( "Killing 3 ONOS nodes" )
+        killTime = time.time()
+        # TODO: Randomize these nodes or base this on partitions
         # TODO: use threads in this case
-        main.ONOSbench.onosKill( nodes[0].ip_address )
+        killResults = main.ONOSbench.onosKill( nodes[0].ip_address )
         time.sleep( 10 )
-        main.ONOSbench.onosKill( nodes[1].ip_address )
+        killResults = killResults and\
+                      main.ONOSbench.onosKill( nodes[1].ip_address )
         time.sleep( 10 )
-        main.ONOSbench.onosKill( nodes[2].ip_address )
+        killResults = killResults and\
+                      main.ONOSbench.onosKill( nodes[2].ip_address )
+        utilities.assert_equals( expect=main.TRUE, actual=killResults,
+                                 onpass="ONOS Killed successfully",
+                                 onfail="ONOS kill NOT successful" )
 
         main.step( "Checking if ONOS is up yet" )
         count = 0
@@ -1417,19 +1595,39 @@
             onosIsupResult = onos1Isup and onos2Isup and onos3Isup
             count = count + 1
         # TODO: if it becomes an issue, we can retry this step  a few times
+        utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+                                 onpass="ONOS restarted successfully",
+                                 onfail="ONOS restart NOT successful" )
 
+        main.step( "Restarting ONOS CLIs" )
         cliResult1 = main.ONOScli1.startOnosCli( nodes[0].ip_address )
         cliResult2 = main.ONOScli2.startOnosCli( nodes[1].ip_address )
         cliResult3 = main.ONOScli3.startOnosCli( nodes[2].ip_address )
         cliResults = cliResult1 and cliResult2 and cliResult3
+        utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                 onpass="ONOS cli restarted",
+                                 onfail="ONOS cli did not restart" )
 
         # Grab the time of restart so we chan check how long the gossip
         # protocol has had time to work
-        main.restartTime = time.time()
-        caseResults = main.TRUE and onosIsupResult and cliResults
-        utilities.assert_equals( expect=main.TRUE, actual=caseResults,
-                                 onpass="ONOS restart successful",
-                                 onfail="ONOS restart NOT successful" )
+        main.restartTime = time.time() - killTime
+        main.log.debug( "Restart time: " + str( main.restartTime ) )
+        '''
+        # FIXME: revisit test plan for election with madan
+        # Rerun for election on restarted nodes
+        run1 = CLIs[0].electionTestRun()
+        run2 = CLIs[1].electionTestRun()
+        run3 = CLIs[2].electionTestRun()
+        runResults = run1 and run2 and run3
+        utilities.assert_equals( expect=main.TRUE, actual=runResults,
+                                 onpass="Reran for election",
+                                 onfail="Failed to rerun for election" )
+        '''
+        # TODO: MAke this configurable. Also, we are breaking the above timer
+        time.sleep( 60 )
+        main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+        main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+        main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
 
     def CASE7( self, main ):
         """
@@ -1463,8 +1661,8 @@
             onpass="Each device has a master",
             onfail="Some devices don't have a master assigned" )
 
+        main.step( "Read device roles from ONOS" )
         ONOSMastership = []
-        mastershipCheck = main.FALSE
         consistentMastership = True
         rolesResults = True
         threads = []
@@ -1481,7 +1679,7 @@
 
         for i in range( numControllers ):
             if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " roles" )
                 main.log.warn(
                     "ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1495,7 +1693,7 @@
 
         main.step( "Check for consistency in roles from each controller" )
         if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
-            main.log.report(
+            main.log.info(
                 "Switch roles are consistent across all ONOS nodes" )
         else:
             consistentMastership = False
@@ -1514,9 +1712,9 @@
                         sort_keys=True,
                         indent=4,
                         separators=( ',', ': ' ) ) )
-        elif rolesResults and not consistentMastership:
-            mastershipCheck = main.TRUE
 
+        # NOTE: we expect mastership to change on controller failure
+        '''
         description2 = "Compare switch roles from before failure"
         main.step( description2 )
         try:
@@ -1542,15 +1740,12 @@
             else:
                 main.log.warn( "Mastership of switch %s changed" % switchDPID )
                 mastershipCheck = main.FALSE
-        if mastershipCheck == main.TRUE:
-            main.log.report( "Mastership of Switches was not changed" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=mastershipCheck,
             onpass="Mastership of Switches was not changed",
             onfail="Mastership of some switches changed" )
-        # NOTE: we expect mastership to change on controller failure
-        mastershipCheck = consistentMastership
+        '''
 
         main.step( "Get the intents and compare across all nodes" )
         ONOSIntents = []
@@ -1572,7 +1767,7 @@
 
         for i in range( numControllers ):
             if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " intents" )
                 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
                                repr( ONOSIntents[ i ] ) )
@@ -1585,10 +1780,38 @@
 
         main.step( "Check for consistency in Intents from each controller" )
         if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
-            main.log.report( "Intents are consistent across all ONOS " +
+            main.log.info( "Intents are consistent across all ONOS " +
                              "nodes" )
         else:
             consistentIntents = False
+
+        # Try to make it easy to figure out what is happening
+        #
+        # Intent      ONOS1      ONOS2    ...
+        #  0x01     INSTALLED  INSTALLING
+        #  ...        ...         ...
+        #  ...        ...         ...
+        title = "   ID"
+        for n in range( numControllers ):
+            title += " " * 10 + "ONOS" + str( n + 1 )
+        main.log.warn( title )
+        # get all intent keys in the cluster
+        keys = []
+        for nodeStr in ONOSIntents:
+            node = json.loads( nodeStr )
+            for intent in node:
+                keys.append( intent.get( 'id' ) )
+        keys = set( keys )
+        for key in keys:
+            row = "%-13s" % key
+            for nodeStr in ONOSIntents:
+                node = json.loads( nodeStr )
+                for intent in node:
+                    if intent.get( 'id' ) == key:
+                        row += "%-15s" % intent.get( 'state' )
+            main.log.warn( row )
+        # End table view
+
         utilities.assert_equals(
             expect=True,
             actual=consistentIntents,
@@ -1624,22 +1847,41 @@
         main.step( "Compare current intents with intents before the failure" )
         # NOTE: this requires case 5 to pass for intentState to be set.
         #      maybe we should stop the test if that fails?
-        sameIntents = main.TRUE
+        sameIntents = main.FALSE
         if intentState and intentState == ONOSIntents[ 0 ]:
             sameIntents = main.TRUE
-            main.log.report( "Intents are consistent with before failure" )
+            main.log.info( "Intents are consistent with before failure" )
         # TODO: possibly the states have changed? we may need to figure out
         #       what the acceptable states are
-        else:
+        elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+            sameIntents = main.TRUE
             try:
-                main.log.warn( "ONOS intents: " )
-                main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
-                                           sort_keys=True, indent=4,
-                                           separators=( ',', ': ' ) ) )
+                before = json.loads( intentState )
+                after = json.loads( ONOSIntents[ 0 ] )
+                for intent in before:
+                    if intent not in after:
+                        sameIntents = main.FALSE
+                        main.log.debug( "Intent is not currently in ONOS " +\
+                                        "(at least in the same form):" )
+                        main.log.debug( json.dumps( intent ) )
             except ( ValueError, TypeError ):
                 main.log.exception( "Exception printing intents" )
-                main.log.warn( repr( ONOSIntents[0] ) )
-            sameIntents = main.FALSE
+                main.log.debug( repr( ONOSIntents[0] ) )
+                main.log.debug( repr( intentState ) )
+        if sameIntents == main.FALSE:
+            try:
+                main.log.debug( "ONOS intents before: " )
+                main.log.debug( json.dumps( json.loads( intentState ),
+                                            sort_keys=True, indent=4,
+                                            separators=( ',', ': ' ) ) )
+                main.log.debug( "Current ONOS intents: " )
+                main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+                                            sort_keys=True, indent=4,
+                                            separators=( ',', ': ' ) ) )
+            except ( ValueError, TypeError ):
+                main.log.exception( "Exception printing intents" )
+                main.log.debug( repr( ONOSIntents[0] ) )
+                main.log.debug( repr( intentState ) )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=sameIntents,
@@ -1662,8 +1904,6 @@
             if FlowTables == main.FALSE:
                 main.log.info( "Differences in flow table for switch: s" +
                                str( i + 1 ) )
-        if FlowTables == main.TRUE:
-            main.log.report( "No changes were found in the flow tables" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=FlowTables,
@@ -1689,36 +1929,37 @@
             main.log.info( "There are multiple mininet process running" )
         elif LossInPings == main.FALSE:
             main.log.info( "No Loss in the pings" )
-            main.log.report( "No loss of dataplane connectivity" )
+            main.log.info( "No loss of dataplane connectivity" )
         utilities.assert_equals(
             expect=main.FALSE,
             actual=LossInPings,
             onpass="No Loss of connectivity",
             onfail="Loss of dataplane connectivity detected" )
 
+        main.step( "Leadership Election is still functional" )
         # Test of LeadershipElection
         leaderList = []
         # FIXME: make sure this matches nodes that were restarted
         restarted = [ nodes[0].ip_address, nodes[1].ip_address,
                       nodes[2].ip_address ]
-            
+
         leaderResult = main.TRUE
         for cli in CLIs:
             leaderN = cli.electionTestLeader()
             leaderList.append( leaderN )
             if leaderN == main.FALSE:
-                # error in  response
-                main.log.report( "Something is wrong with " +
+                # error in response
+                main.log.error( "Something is wrong with " +
                                  "electionTestLeader function, check the" +
                                  " error logs" )
                 leaderResult = main.FALSE
             elif leaderN is None:
-                main.log.report( cli.name +
+                main.log.error( cli.name +
                                  " shows no leader for the election-app was" +
                                  " elected after the old one died" )
                 leaderResult = main.FALSE
             elif leaderN in restarted:
-                main.log.report( cli.name + " shows " + str( leaderN ) +
+                main.log.error( cli.name + " shows " + str( leaderN ) +
                                  " as leader for the election-app, but it " +
                                  "was restarted" )
                 leaderResult = main.FALSE
@@ -1727,25 +1968,12 @@
             main.log.error(
                 "Inconsistent view of leader for the election test app" )
             # TODO: print the list
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a new " +
-                             "leader was re-elected if applicable )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
             onpass="Leadership election passed",
             onfail="Something went wrong with Leadership election" )
 
-        result = ( mastershipCheck and intentCheck and FlowTables and
-                   ( not LossInPings ) and rolesNotNull and leaderResult )
-        result = int( result )
-        if result == main.TRUE:
-            main.log.report( "Constant State Tests Passed" )
-        utilities.assert_equals( expect=main.TRUE, actual=result,
-                                 onpass="Constant State Tests Passed",
-                                 onfail="Constant state tests failed" )
-
     def CASE8( self, main ):
         """
         Compare topo
@@ -1763,15 +1991,24 @@
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
 
-        description = "Compare ONOS Topology view to Mininet topology"
-        main.case( description )
-        main.log.report( description )
+        main.case( "Compare ONOS Topology view to Mininet topology" )
+        main.caseExplaination = "Compare topology objects between Mininet" +\
+                                " and ONOS"
         main.step( "Create TestONTopology object" )
-        ctrls = []
-        for node in nodes:
-            temp = ( node, node.name, node.ip_address, 6633 )
-            ctrls.append( temp )
-        MNTopo = TestONTopology( main.Mininet1, ctrls )
+        try:
+            ctrls = []
+            for node in nodes:
+                temp = ( node, node.name, node.ip_address, 6633 )
+                ctrls.append( temp )
+            MNTopo = TestONTopology( main.Mininet1, ctrls )
+        except Exception:
+            objResult = main.FALSE
+        else:
+            objResult = main.TRUE
+        utilities.assert_equals( expect=main.TRUE, actual=objResult,
+                                 onpass="Created TestONTopology object",
+                                 onfail="Exception while creating " +
+                                        "TestONTopology object" )
 
         main.step( "Comparing ONOS topology to MN" )
         devicesResults = main.TRUE
@@ -1822,9 +2059,9 @@
             for controller in range( 0, len( hosts ) ):
                 controllerStr = str( controller + 1 )
                 for host in hosts[ controller ]:
-                    if host is None or host.get( 'ips', [] ) == []:
+                    if host is None or host.get( 'ipAddresses', [] ) == []:
                         main.log.error(
-                            "DEBUG:Error with host ips on controller" +
+                            "DEBUG:Error with host ipAddresses on controller" +
                             controllerStr + ": " + str( host ) )
                         ipResult = main.FALSE
             ports = []
@@ -1930,6 +2167,7 @@
             # Compare json objects for hosts and dataplane clusters
 
             # hosts
+            main.step( "Hosts view is consistent across all ONOS nodes" )
             consistentHostsResult = main.TRUE
             for controller in range( len( hosts ) ):
                 controllerStr = str( controller + 1 )
@@ -1937,13 +2175,13 @@
                     if hosts[ controller ] == hosts[ 0 ]:
                         continue
                     else:  # hosts not consistent
-                        main.log.report( "hosts from ONOS" + controllerStr +
+                        main.log.error( "hosts from ONOS" + controllerStr +
                                          " is inconsistent with ONOS1" )
                         main.log.warn( repr( hosts[ controller ] ) )
                         consistentHostsResult = main.FALSE
 
                 else:
-                    main.log.report( "Error in getting ONOS hosts from ONOS" +
+                    main.log.error( "Error in getting ONOS hosts from ONOS" +
                                      controllerStr )
                     consistentHostsResult = main.FALSE
                     main.log.warn( "ONOS" + controllerStr +
@@ -1956,6 +2194,7 @@
                 onfail="ONOS nodes have different views of hosts" )
 
             # Strongly connected clusters of devices
+            main.step( "Clusters view is consistent across all ONOS nodes" )
             consistentClustersResult = main.TRUE
             for controller in range( len( clusters ) ):
                 controllerStr = str( controller + 1 )
@@ -1963,13 +2202,13 @@
                     if clusters[ controller ] == clusters[ 0 ]:
                         continue
                     else:  # clusters not consistent
-                        main.log.report( "clusters from ONOS" +
+                        main.log.error( "clusters from ONOS" +
                                          controllerStr +
                                          " is inconsistent with ONOS1" )
                         consistentClustersResult = main.FALSE
 
                 else:
-                    main.log.report( "Error in getting dataplane clusters " +
+                    main.log.error( "Error in getting dataplane clusters " +
                                      "from ONOS" + controllerStr )
                     consistentClustersResult = main.FALSE
                     main.log.warn( "ONOS" + controllerStr +
@@ -1981,6 +2220,8 @@
                 onpass="Clusters view is consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of clusters" )
             # there should always only be one cluster
+            main.step( "Topology view is correct and consistent across all " +
+                       "ONOS nodes" )
             try:
                 numClusters = len( json.loads( clusters[ 0 ] ) )
             except ( ValueError, TypeError ):
@@ -2011,12 +2252,11 @@
         utilities.assert_equals( expect=main.TRUE, actual=topoResult,
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
-        if topoResult == main.TRUE:
-            main.log.report( "ONOS topology view matches Mininet topology" )
 
         # FIXME: move this to an ONOS state case
         main.step( "Checking ONOS nodes" )
         nodesOutput = []
+        nodeResults = main.TRUE
         threads = []
         for i in range( numControllers ):
             t = main.Thread( target=CLIs[i].nodes,
@@ -2033,9 +2273,10 @@
             try:
                 current = json.loads( i )
                 for node in current:
+                    currentResult = main.FALSE
                     if node['ip'] in ips:  # node in nodes() output is in cell
                         if node['state'] == 'ACTIVE':
-                            pass  # as it should be
+                            currentResult = main.TRUE
                         else:
                             main.log.error( "Error in ONOS node availability" )
                             main.log.error(
@@ -2044,9 +2285,13 @@
                                                 indent=4,
                                                 separators=( ',', ': ' ) ) )
                             break
+                    nodeResults = nodeResults and currentResult
             except ( ValueError, TypeError ):
                 main.log.error( "Error parsing nodes output" )
                 main.log.warn( repr( i ) )
+        utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
 
     def CASE9( self, main ):
         """
@@ -2064,7 +2309,6 @@
 
         description = "Turn off a link to ensure that Link Discovery " +\
                       "is working properly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Kill Link between s3 and s28" )
@@ -2093,7 +2337,6 @@
 
         description = "Restore a link to ensure that Link Discovery is " + \
                       "working properly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Bring link between s3 and s28 back up" )
@@ -2121,14 +2364,13 @@
         switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
 
         description = "Killing a switch to ensure it is discovered correctly"
-        main.log.report( description )
         main.case( description )
         switch = main.params[ 'kill' ][ 'switch' ]
         switchDPID = main.params[ 'kill' ][ 'dpid' ]
 
         # TODO: Make this switch parameterizable
         main.step( "Kill " + switch )
-        main.log.report( "Deleting " + switch )
+        main.log.info( "Deleting " + switch )
         main.Mininet1.delSwitch( switch )
         main.log.info( "Waiting " + str( switchSleep ) +
                        " seconds for switch down to be discovered" )
@@ -2167,11 +2409,9 @@
         switchDPID = main.params[ 'kill' ][ 'dpid' ]
         links = main.params[ 'kill' ][ 'links' ].split()
         description = "Adding a switch to ensure it is discovered correctly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Add back " + switch )
-        main.log.report( "Adding back " + switch )
         main.Mininet1.addSwitch( switch, dpid=switchDPID )
         for peer in links:
             main.Mininet1.addLink( switch, peer )
@@ -2220,9 +2460,7 @@
         colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
                    'blue': '\033[94m', 'green': '\033[92m',
                    'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
-        description = "Test Cleanup"
-        main.log.report( description )
-        main.case( description )
+        main.case( "Test Cleanup" )
         main.step( "Killing tcpdumps" )
         main.Mininet2.stopTcpdump()
 
@@ -2269,26 +2507,84 @@
         time.sleep( 10 )
 
         main.step( "Stopping Mininet" )
-        main.Mininet1.stopNet()
+        mnResult = main.Mininet1.stopNet()
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet stopped",
+                                 onfail="MN cleanup NOT successful" )
 
         main.step( "Checking ONOS Logs for errors" )
         for node in nodes:
             print colors[ 'purple' ] + "Checking logs for errors on " + \
                 node.name + ":" + colors[ 'end' ]
-            print main.ONOSbench.checkLogs( node.ip_address )
+            print main.ONOSbench.checkLogs( node.ip_address, restart=True )
 
         main.step( "Packing and rotating pcap archives" )
         os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
 
-        # TODO: actually check something here
-        utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
-                                 onpass="Test cleanup successful",
-                                 onfail="Test cleanup NOT successful" )
+        try:
+            timerLog = open( main.logdir + "/Timers.csv", 'w')
+            # Overwrite with empty line and close
+            labels = "Gossip Intents, Restart"
+            data = str( gossipTime ) + ", " + str( main.restartTime )
+            timerLog.write( labels + "\n" + data )
+            timerLog.close()
+        except NameError, e:
+            main.log.exception(e)
 
     def CASE14( self, main ):
         """
         start election app on all onos nodes
         """
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+
+        main.case("Start Leadership Election app")
+        main.step( "Install leadership election app" )
+        appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=appResult,
+            onpass="Election app installed",
+            onfail="Something went wrong with installing Leadership election" )
+
+        main.step( "Run for election on each node" )
+        leaderResult = main.TRUE
+        leaders = []
+        for cli in CLIs:
+            cli.electionTestRun()
+        for cli in CLIs:
+            leader = cli.electionTestLeader()
+            if leader is None or leader == main.FALSE:
+                main.log.error( cli.name + ": Leader for the election app " +
+                                 "should be an ONOS node, instead got '" +
+                                 str( leader ) + "'" )
+                leaderResult = main.FALSE
+            leaders.append( leader )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=leaderResult,
+            onpass="Successfully ran for leadership",
+            onfail="Failed to run for leadership" )
+
+        main.step( "Check that each node shows the same leader" )
+        sameLeader = main.TRUE
+        if len( set( leaders ) ) != 1:
+            sameLeader = main.FALSE
+            main.log.error( "Results of electionTestLeader is order of CLIs:" +
+                            str( leaders ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=sameLeader,
+            onpass="Leadership is consistent for the election topic",
+            onfail="Nodes have different leaders" )
+
+    def CASE15( self, main ):
+        """
+        Check that Leadership Election is still functional
+        """
         import time
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
@@ -2297,51 +2593,31 @@
         assert nodes, "nodes not defined"
 
         leaderResult = main.TRUE
-        main.log.info( "Install leadership election app" )
-        main.ONOScli1.activateApp( "org.onosproject.election" )
+        description = "Check that Leadership Election is still functional"
+        main.case( description )
+
+        main.step( "Check that each node shows the same leader" )
+        sameLeader = main.TRUE
         leaders = []
         for cli in CLIs:
             leader = cli.electionTestLeader()
-            if leader is None or leader == main.FALSE:
-                main.log.report( cli.name + ": Leader for the election app " +
-                                 "should be an ONOS node, instead got '" +
-                                 str( leader ) + "'" )
-                leaderResult = main.FALSE
             leaders.append( leader )
         if len( set( leaders ) ) != 1:
-            leaderResult = main.FALSE
+            sameLeader = main.FALSE
             main.log.error( "Results of electionTestLeader is order of CLIs:" +
                             str( leaders ) )
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a leader " +
-                             "was elected )" )
         utilities.assert_equals(
             expect=main.TRUE,
-            actual=leaderResult,
-            onpass="Leadership election passed",
-            onfail="Something went wrong with Leadership election" )
+            actual=sameLeader,
+            onpass="Leadership is consistent for the election topic",
+            onfail="Nodes have different leaders" )
 
-    def CASE15( self, main ):
-        """
-        Check that Leadership Election is still functional
-        """
-        assert numControllers, "numControllers not defined"
-        assert main, "main not defined"
-        assert utilities.assert_equals, "utilities.assert_equals not defined"
-        assert CLIs, "CLIs not defined"
-        assert nodes, "nodes not defined"
-
-        leaderResult = main.TRUE
-        description = "Check that Leadership Election is still functional"
-        main.log.report( description )
-        main.case( description )
         main.step( "Find current leader and withdraw" )
         leader = main.ONOScli1.electionTestLeader()
         # do some sanity checking on leader before using it
         withdrawResult = main.FALSE
         if leader is None or leader == main.FALSE:
-            main.log.report(
+            main.log.error(
                 "Leader for the election app should be an ONOS node," +
                 "instead got '" + str( leader ) + "'" )
             leaderResult = main.FALSE
@@ -2357,8 +2633,8 @@
         utilities.assert_equals(
             expect=main.TRUE,
             actual=withdrawResult,
-            onpass="App was withdrawn from election",
-            onfail="App was not withdrawn from election" )
+            onpass="Node was withdrawn from election",
+            onfail="Node was not withdrawn from election" )
 
         main.step( "Make sure new leader is elected" )
         # FIXME: use threads
@@ -2367,19 +2643,20 @@
             leaderN = cli.electionTestLeader()
             leaderList.append( leaderN )
             if leaderN == leader:
-                main.log.report(  cli.name + " still sees " + str( leader ) +
+                main.log.error(  cli.name + " still sees " + str( leader ) +
                                   " as leader after they withdrew" )
                 leaderResult = main.FALSE
             elif leaderN == main.FALSE:
                 # error in  response
                 # TODO: add check for "Command not found:" in the driver, this
                 #       means the app isn't loaded
-                main.log.report( "Something is wrong with " +
+                main.log.error( "Something is wrong with " +
                                  "electionTestLeader function, " +
                                  "check the error logs" )
                 leaderResult = main.FALSE
             elif leaderN is None:
                 # node may not have recieved the event yet
+                time.sleep(7)
                 leaderN = cli.electionTestLeader()
                 leaderList.pop()
                 leaderList.append( leaderN )
@@ -2390,17 +2667,12 @@
                            "' as the leader" )
             consistentLeader = main.TRUE
         else:
-            main.log.report(
+            main.log.error(
                 "Inconsistent responses for leader of Election-app:" )
             for n in range( len( leaderList ) ):
-                main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+                main.log.error( "ONOS" + str( n + 1 ) + " response: " +
                                  str( leaderList[ n ] ) )
         leaderResult = leaderResult and consistentLeader
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a new " +
-                             "leader was elected when the old leader " +
-                             "resigned )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
@@ -2418,18 +2690,1110 @@
             actual=runResult,
             onpass="App re-ran for election",
             onfail="App failed to run for election" )
-        if consistentLeader == main.TRUE:
-            afterRun = main.ONOScli1.electionTestLeader()
-            # verify leader didn't just change
-            if afterRun == leaderList[ 0 ]:
-                leaderResult = main.TRUE
-            else:
-                leaderResult = main.FALSE
-        # TODO: assert on  run and withdraw results?
+
+        main.step( "Leader did not change when old leader re-ran" )
+        afterRun = main.ONOScli1.electionTestLeader()
+        # verify leader didn't just change
+        if afterRun == leaderList[ 0 ]:
+            afterResult = main.TRUE
+        else:
+            afterResult = main.FALSE
 
         utilities.assert_equals(
             expect=main.TRUE,
-            actual=leaderResult,
-            onpass="Leadership election passed",
+            actual=afterResult,
+            onpass="Old leader successfully re-ran for election",
             onfail="Something went wrong with Leadership election after " +
                    "the old leader re-ran for election" )
+
+    def CASE16( self, main ):
+        """
+        Install Distributed Primitives app
+        """
+        import time
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+
+        # Variables for the distributed primitives tests
+        global pCounterName
+        global iCounterName
+        global pCounterValue
+        global iCounterValue
+        global onosSet
+        global onosSetName
+        pCounterName = "TestON-Partitions"
+        iCounterName = "TestON-inMemory"
+        pCounterValue = 0
+        iCounterValue = 0
+        onosSet = set([])
+        onosSetName = "TestON-set"
+
+        description = "Install Primitives app"
+        main.case( description )
+        main.step( "Install Primitives app" )
+        appName = "org.onosproject.distributedprimitives"
+        appResults = CLIs[0].activateApp( appName )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=appResults,
+                                 onpass="Primitives app activated",
+                                 onfail="Primitives app not activated" )
+        time.sleep( 5 )  # To allow all nodes to activate
+
+    def CASE17( self, main ):
+        """
+        Check for basic functionality with distributed primitives
+        """
+        # Make sure variables are defined/set
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+        assert pCounterName, "pCounterName not defined"
+        assert iCounterName, "iCounterName not defined"
+        assert onosSetName, "onosSetName not defined"
+        # NOTE: assert fails if value is 0/None/Empty/False
+        try:
+            pCounterValue
+        except NameError:
+            main.log.error( "pCounterValue not defined, setting to 0" )
+            pCounterValue = 0
+        try:
+            iCounterValue
+        except NameError:
+            main.log.error( "iCounterValue not defined, setting to 0" )
+            iCounterValue = 0
+        try:
+            onosSet
+        except NameError:
+            main.log.error( "onosSet not defined, setting to empty Set" )
+            onosSet = set([])
+        # Variables for the distributed primitives tests. These are local only
+        addValue = "a"
+        addAllValue = "a b c d e f"
+        retainValue = "c d e f"
+
+        description = "Check for basic functionality with distributed " +\
+                      "primitives"
+        main.case( description )
+        main.caseExplaination = "Test the methods of the distributed primitives (counters and sets) throught the cli"
+        # DISTRIBUTED ATOMIC COUNTERS
+        main.step( "Increment and get a default counter on each node" )
+        pCounters = []
+        threads = []
+        addedPValues = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counterTestIncrement,
+                             name="counterIncrement-" + str( i ),
+                             args=[ pCounterName ] )
+            pCounterValue += 1
+            addedPValues.append( pCounterValue )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            pCounters.append( t.result )
+        # Check that counter incremented numController times
+        pCounterResults = True
+        for i in addedPValues:
+            tmpResult = i in pCounters
+            pCounterResults = pCounterResults and tmpResult
+            if not tmpResult:
+                main.log.error( str( i ) + " is not in partitioned "
+                                "counter incremented results" )
+        utilities.assert_equals( expect=True,
+                                 actual=pCounterResults,
+                                 onpass="Default counter incremented",
+                                 onfail="Error incrementing default" +
+                                        " counter" )
+
+        main.step( "Increment and get an in memory counter on each node" )
+        iCounters = []
+        addedIValues = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counterTestIncrement,
+                             name="icounterIncrement-" + str( i ),
+                             args=[ iCounterName ],
+                             kwargs={ "inMemory": True } )
+            iCounterValue += 1
+            addedIValues.append( iCounterValue )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            iCounters.append( t.result )
+        # Check that counter incremented numController times
+        iCounterResults = True
+        for i in addedIValues:
+            tmpResult = i in iCounters
+            iCounterResults = iCounterResults and tmpResult
+            if not tmpResult:
+                main.log.error( str( i ) + " is not in the in-memory "
+                                "counter incremented results" )
+        utilities.assert_equals( expect=True,
+                                 actual=iCounterResults,
+                                 onpass="In memory counter incremented",
+                                 onfail="Error incrementing in memory" +
+                                        " counter" )
+
+        main.step( "Check counters are consistant across nodes" )
+        onosCounters = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counters,
+                             name="counters-" + str( i ) )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            onosCounters.append( t.result )
+        tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+        if all( tmp ):
+            main.log.info( "Counters are consistent across all nodes" )
+            consistentCounterResults = main.TRUE
+        else:
+            main.log.error( "Counters are not consistent across all nodes" )
+            consistentCounterResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=consistentCounterResults,
+                                 onpass="ONOS counters are consistent " +
+                                        "across nodes",
+                                 onfail="ONOS Counters are inconsistent " +
+                                        "across nodes" )
+
+        main.step( "Counters we added have the correct values" )
+        correctResults = main.TRUE
+        for i in range( numControllers ):
+            current = json.loads( onosCounters[i] )
+            pValue = None
+            iValue = None
+            try:
+                for database in current:
+                    partitioned = database.get( 'partitionedDatabaseCounters' )
+                    if partitioned:
+                        for value in partitioned:
+                            if value.get( 'name' ) == pCounterName:
+                                pValue = value.get( 'value' )
+                                break
+                    inMemory = database.get( 'inMemoryDatabaseCounters' )
+                    if inMemory:
+                        for value in inMemory:
+                            if value.get( 'name' ) == iCounterName:
+                                iValue = value.get( 'value' )
+                                break
+            except AttributeError, e:
+                main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
+                                "is not as expected" )
+                correctResults = main.FALSE
+            if pValue == pCounterValue:
+                main.log.info( "Partitioned counter value is correct" )
+            else:
+                main.log.error( "Partitioned counter value is incorrect," +
+                                " expected value: " + str( pCounterValue )
+                                + " current value: " + str( pValue ) )
+                correctResults = main.FALSE
+            if iValue == iCounterValue:
+                main.log.info( "In memory counter value is correct" )
+            else:
+                main.log.error( "In memory counter value is incorrect, " +
+                                "expected value: " + str( iCounterValue ) +
+                                " current value: " + str( iValue ) )
+                correctResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=correctResults,
+                                 onpass="Added counters are correct",
+                                 onfail="Added counters are incorrect" )
+        # DISTRIBUTED SETS
+        main.step( "Distributed Set get" )
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=getResults,
+                                 onpass="Set elements are correct",
+                                 onfail="Set elements are incorrect" )
+
+        main.step( "Distributed Set size" )
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=sizeResults,
+                                 onpass="Set sizes are correct",
+                                 onfail="Set sizes are incorrect" )
+
+        main.step( "Distributed Set add()" )
+        onosSet.add( addValue )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAdd-" + str( i ),
+                             args=[ onosSetName, addValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addResults = main.FALSE
+            else:
+                # unexpected result
+                addResults = main.FALSE
+        if addResults != main.TRUE:
+            main.log.error( "Error executing set add" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addResults = addResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addResults,
+                                 onpass="Set add correct",
+                                 onfail="Set add was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set contains()" )
+        containsResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setContains-" + str( i ),
+                             args=[ onosSetName ],
+                             kwargs={ "values": addValue } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            # NOTE: This is the tuple
+            containsResponses.append( t.result )
+
+        containsResults = main.TRUE
+        for i in range( numControllers ):
+            if containsResponses[ i ] == main.ERROR:
+                containsResults = main.FALSE
+            else:
+                containsResults = containsResults and\
+                                  containsResponses[ i ][ 1 ]
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=containsResults,
+                                 onpass="Set contains is functional",
+                                 onfail="Set contains failed" )
+
+        main.step( "Distributed Set containsAll()" )
+        containsAllResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setContainsAll-" + str( i ),
+                             args=[ onosSetName ],
+                             kwargs={ "values": addAllValue } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            # NOTE: This is the tuple
+            containsAllResponses.append( t.result )
+
+        containsAllResults = main.TRUE
+        for i in range( numControllers ):
+            if containsResponses[ i ] == main.ERROR:
+                containsResults = main.FALSE
+            else:
+                containsResults = containsResults and\
+                                  containsResponses[ i ][ 1 ]
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=containsAllResults,
+                                 onpass="Set containsAll is functional",
+                                 onfail="Set containsAll failed" )
+
+        main.step( "Distributed Set remove()" )
+        onosSet.remove( addValue )
+        removeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestRemove-" + str( i ),
+                             args=[ onosSetName, addValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            removeResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        removeResults = main.TRUE
+        for i in range( numControllers ):
+            if removeResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif removeResponses[ i ] == main.FALSE:
+                # not in set, probably fine
+                pass
+            elif removeResponses[ i ] == main.ERROR:
+                # Error in execution
+                removeResults = main.FALSE
+            else:
+                # unexpected result
+                removeResults = main.FALSE
+        if removeResults != main.TRUE:
+            main.log.error( "Error executing set remove" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        removeResults = removeResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=removeResults,
+                                 onpass="Set remove correct",
+                                 onfail="Set remove was incorrect" )
+
+        main.step( "Distributed Set removeAll()" )
+        onosSet.difference_update( addAllValue.split() )
+        removeAllResponses = []
+        threads = []
+        try:
+            for i in range( numControllers ):
+                t = main.Thread( target=CLIs[i].setTestRemove,
+                                 name="setTestRemoveAll-" + str( i ),
+                                 args=[ onosSetName, addAllValue ] )
+                threads.append( t )
+                t.start()
+            for t in threads:
+                t.join()
+                removeAllResponses.append( t.result )
+        except Exception, e:
+            main.log.exception(e)
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        removeAllResults = main.TRUE
+        for i in range( numControllers ):
+            if removeAllResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif removeAllResponses[ i ] == main.FALSE:
+                # not in set, probably fine
+                pass
+            elif removeAllResponses[ i ] == main.ERROR:
+                # Error in execution
+                removeAllResults = main.FALSE
+            else:
+                # unexpected result
+                removeAllResults = main.FALSE
+        if removeAllResults != main.TRUE:
+            main.log.error( "Error executing set removeAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        removeAllResults = removeAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=removeAllResults,
+                                 onpass="Set removeAll correct",
+                                 onfail="Set removeAll was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set clear()" )
+        onosSet.clear()
+        clearResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestClear-" + str( i ),
+                             args=[ onosSetName, " "],  # Values doesn't matter
+                             kwargs={ "clear": True } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            clearResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        clearResults = main.TRUE
+        for i in range( numControllers ):
+            if clearResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif clearResponses[ i ] == main.FALSE:
+                # Nothing set, probably fine
+                pass
+            elif clearResponses[ i ] == main.ERROR:
+                # Error in execution
+                clearResults = main.FALSE
+            else:
+                # unexpected result
+                clearResults = main.FALSE
+        if clearResults != main.TRUE:
+            main.log.error( "Error executing set clear" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        clearResults = clearResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=clearResults,
+                                 onpass="Set clear correct",
+                                 onfail="Set clear was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set retain()" )
+        onosSet.intersection_update( retainValue.split() )
+        retainResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestRetain-" + str( i ),
+                             args=[ onosSetName, retainValue ],
+                             kwargs={ "retain": True } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            retainResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        retainResults = main.TRUE
+        for i in range( numControllers ):
+            if retainResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif retainResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif retainResponses[ i ] == main.ERROR:
+                # Error in execution
+                retainResults = main.FALSE
+            else:
+                # unexpected result
+                retainResults = main.FALSE
+        if retainResults != main.TRUE:
+            main.log.error( "Error executing set retain" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " +
+                                str( size ) + " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        retainResults = retainResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=retainResults,
+                                 onpass="Set retain correct",
+                                 onfail="Set retain was incorrect" )
+
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
index e1421cd..f82aba3 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
@@ -25,6 +25,7 @@
     <Git> False </Git>
     <branch> master </branch>
     <num_controllers> 7 </num_controllers>
+    <tcpdump> False </tcpdump>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
index 3c4541c..d2a2ec1 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
@@ -120,12 +120,27 @@
             main.ONOSbench.gitCheckout( gitBranch )
             gitPullResult = main.ONOSbench.gitPull()
 
-            main.step( "Using mvn clean & install" )
+            main.step( "Using mvn clean and install" )
             cleanInstallResult = main.ONOSbench.cleanInstall()
         else:
             main.log.warn( "Did not pull new code so skipping mvn " +
                            "clean install" )
         main.ONOSbench.getVersion( report=True )
+        # GRAPHS
+        # NOTE: important params here:
+        #       job = name of Jenkins job
+        #       Plot Name = Plot-HA, only can be used if multiple plots
+        #       index = The number of the graph under plot name
+        job = "HANetworkPartition"
+        graphs = '<ac:structured-macro ac:name="html">\n'
+        graphs += '<ac:plain-text-body><![CDATA[\n'
+        graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+                  '/plot/getPlot?index=0&width=500&height=300"' +\
+                  'noborder="0" width="500" height="300" scrolling="yes" '+\
+                  'seamless="seamless"></iframe>\n'
+        graphs += ']]></ac:plain-text-body>\n'
+        graphs += '</ac:structured-macro>\n'
+        main.log.wiki(graphs)
 
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()
@@ -202,12 +217,13 @@
         cliResults = cliResult1 and cliResult2 and cliResult3 and\
             cliResult4 and cliResult5 and cliResult6 and cliResult7
 
-        main.step( "Start Packet Capture MN" )
-        main.Mininet2.startTcpdump(
-            str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
-            + "-MN.pcap",
-            intf=main.params[ 'MNtcpdump' ][ 'intf' ],
-            port=main.params[ 'MNtcpdump' ][ 'port' ] )
+        if main.params[ 'tcpdump' ].lower() == "true":
+            main.step( "Start Packet Capture MN" )
+            main.Mininet2.startTcpdump(
+                str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+                + "-MN.pcap",
+                intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+                port=main.params[ 'MNtcpdump' ][ 'port' ] )
 
         case1Result = ( cleanInstallResult and packageResult and
                         cellResult and verifyResult and onosInstallResult
@@ -218,6 +234,7 @@
                                  onfail="Test startup NOT successful" )
 
         if case1Result == main.FALSE:
+            main.log.error( "Failed to start ONOS, stopping test" )
             main.cleanup()
             main.exit()
 
@@ -1414,7 +1431,7 @@
             for controller in range( 0, len( hosts ) ):
                 controllerStr = str( controller + 1 )
                 for host in hosts[ controller ]:
-                    if host[ 'ips' ] == []:
+                    if host[ 'ipAddresses' ] == []:
                         main.log.error(
                             "DEBUG:Error with host ips on controller" +
                             controllerStr + ": " + str( host ) )
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index ee4374d..9106d68 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,11 +1,31 @@
 <PARAMS>
-    <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    #List of test cases:
+    #CASE1: Compile ONOS and push it to the test machines
+    #CASE2: Assign mastership to controllers
+    #CASE3: Assign intents
+    #CASE4: Ping across added host intents
+    #CASE5: Reading state of ONOS
+    #CASE6: The Failure case. Since this is the Sanity test, we do nothing.
+    #CASE7: Check state after control plane failure
+    #CASE8: Compare topo
+    #CASE9: Link s3-s28 down
+    #CASE10: Link s3-s28 up
+    #CASE11: Switch down
+    #CASE12: Switch up
+    #CASE13: Clean up
+    #CASE14: start election app on all onos nodes
+    #CASE15: Check that Leadership Election is still functional
+    #CASE16: Install Distributed Primitives app
+    #CASE17: Check for basic functionality with distributed primitives
+    #1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13
+    <testcases>1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
         <cellName>HA</cellName>
     </ENV>
     <Git> True </Git>
     <branch> master </branch>
     <num_controllers> 7 </num_controllers>
+    <tcpdump> False </tcpdump>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 20f6cc8..60c438e 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -19,6 +19,8 @@
 CASE13: Clean up
 CASE14: start election app on all onos nodes
 CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
 """
 
 
@@ -45,8 +47,11 @@
         start cli sessions
         start tcpdump
         """
-        main.log.report( "ONOS HA Sanity test - initialization" )
+        main.log.info( "ONOS HA Sanity test - initialization" )
         main.case( "Setting up test environment" )
+        main.caseExplaination = "Setup the test environment including " +\
+                                "installing ONOS, starting Mininet and ONOS" +\
+                                "cli sessions."
         # TODO: save all the timers and output them for plotting
 
         # load some variables from the params file
@@ -90,36 +95,73 @@
         verifyResult = main.ONOSbench.verifyCell()
 
         # FIXME:this is short term fix
-        main.log.report( "Removing raft logs" )
+        main.log.info( "Removing raft logs" )
         main.ONOSbench.onosRemoveRaftLogs()
 
-        main.log.report( "Uninstalling ONOS" )
+        main.log.info( "Uninstalling ONOS" )
         for node in nodes:
             main.ONOSbench.onosUninstall( node.ip_address )
 
+        # Make sure ONOS is DEAD
+        main.log.info( "Killing any ONOS processes" )
+        killResults = main.TRUE
+        for node in nodes:
+            killed = main.ONOSbench.onosKill( node.ip_address )
+            killResults = killResults and killed
+
         cleanInstallResult = main.TRUE
         gitPullResult = main.TRUE
 
         main.step( "Starting Mininet" )
-        main.Mininet1.startNet( )
+        mnResult = main.Mininet1.startNet( )
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet Started",
+                                 onfail="Error starting Mininet" )
 
-        main.step( "Compiling the latest version of ONOS" )
+        main.step( "Git checkout and pull " + gitBranch )
         if PULLCODE:
-            main.step( "Git checkout and pull " + gitBranch )
             main.ONOSbench.gitCheckout( gitBranch )
             gitPullResult = main.ONOSbench.gitPull()
-            if gitPullResult == main.ERROR:
-                main.log.error( "Error pulling git branch" )
+            # values of 1 or 3 are good
+            utilities.assert_lesser( expect=0, actual=gitPullResult,
+                                      onpass="Git pull successful",
+                                      onfail="Git pull failed" )
+        main.ONOSbench.getVersion( report=True )
 
-            main.step( "Using mvn clean & install" )
+        main.step( "Using mvn clean install" )
+        cleanInstallResult = main.TRUE
+        if PULLCODE and gitPullResult == main.TRUE:
             cleanInstallResult = main.ONOSbench.cleanInstall()
         else:
             main.log.warn( "Did not pull new code so skipping mvn " +
                            "clean install" )
-        main.ONOSbench.getVersion( report=True )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=cleanInstallResult,
+                                 onpass="MCI successful",
+                                 onfail="MCI failed" )
+        # GRAPHS
+        # NOTE: important params here:
+        #       job = name of Jenkins job
+        #       Plot Name = Plot-HA, only can be used if multiple plots
+        #       index = The number of the graph under plot name
+        job = "HASanity"
+        plotName = "Plot-HA"
+        graphs = '<ac:structured-macro ac:name="html">\n'
+        graphs += '<ac:plain-text-body><![CDATA[\n'
+        graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+                  '/plot/' + plotName + '/getPlot?index=0' +\
+                  '&width=500&height=300"' +\
+                  'noborder="0" width="500" height="300" scrolling="yes" ' +\
+                  'seamless="seamless"></iframe>\n'
+        graphs += ']]></ac:plain-text-body>\n'
+        graphs += '</ac:structured-macro>\n'
+        main.log.wiki(graphs)
 
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()
+        utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+                                 onpass="ONOS package successful",
+                                 onfail="ONOS package failed" )
 
         main.step( "Installing ONOS package" )
         onosInstallResult = main.TRUE
@@ -127,6 +169,9 @@
             tmpResult = main.ONOSbench.onosInstall( options="-f",
                                                     node=node.ip_address )
             onosInstallResult = onosInstallResult and tmpResult
+        utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+                                 onpass="ONOS install successful",
+                                 onfail="ONOS install failed" )
 
         main.step( "Checking if ONOS is up yet" )
         for i in range( 2 ):
@@ -134,12 +179,15 @@
             for node in nodes:
                 started = main.ONOSbench.isup( node.ip_address )
                 if not started:
-                    main.log.report( node.name + " didn't start!" )
+                    main.log.error( node.name + " didn't start!" )
                     main.ONOSbench.onosStop( node.ip_address )
                     main.ONOSbench.onosStart( node.ip_address )
                 onosIsupResult = onosIsupResult and started
             if onosIsupResult == main.TRUE:
                 break
+        utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+                                 onpass="ONOS startup successful",
+                                 onfail="ONOS startup failed" )
 
         main.log.step( "Starting ONOS CLI sessions" )
         cliResults = main.TRUE
@@ -154,14 +202,19 @@
         for t in threads:
             t.join()
             cliResults = cliResults and t.result
+        utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                 onpass="ONOS cli startup successful",
+                                 onfail="ONOS cli startup failed" )
 
-        main.step( "Start Packet Capture MN" )
-        main.Mininet2.startTcpdump(
-            str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
-            + "-MN.pcap",
-            intf=main.params[ 'MNtcpdump' ][ 'intf' ],
-            port=main.params[ 'MNtcpdump' ][ 'port' ] )
+        if main.params[ 'tcpdump' ].lower() == "true":
+            main.step( "Start Packet Capture MN" )
+            main.Mininet2.startTcpdump(
+                str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+                + "-MN.pcap",
+                intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+                port=main.params[ 'MNtcpdump' ][ 'port' ] )
 
+        main.step( "App Ids check" )
         appCheck = main.TRUE
         threads = []
         for i in range( numControllers ):
@@ -174,22 +227,15 @@
         for t in threads:
             t.join()
             appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
         if appCheck != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
-        case1Result = ( cleanInstallResult and packageResult and
-                        cellResult and verifyResult and onosInstallResult
-                        and onosIsupResult and cliResults and appCheck)
-
-        utilities.assert_equals( expect=main.TRUE, actual=case1Result,
-                                 onpass="Test startup successful",
-                                 onfail="Test startup NOT successful" )
-
-        if case1Result == main.FALSE:
+        if cliResults == main.FALSE:
+            main.log.error( "Failed to start ONOS, stopping test" )
             main.cleanup()
             main.exit()
 
@@ -198,6 +244,7 @@
         Assign mastership to controllers
         """
         import re
+        import time
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -211,8 +258,12 @@
         assert ONOS6Port, "ONOS6Port not defined"
         assert ONOS7Port, "ONOS7Port not defined"
 
-        main.log.report( "Assigning switches to controllers" )
         main.case( "Assigning Controllers" )
+        main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+                                "and check that an ONOS node becomes the " +\
+                                "master of the device. Then manually assign" +\
+                                " mastership to specific ONOS nodes using" +\
+                                " 'device-role'"
         main.step( "Assign switches to controllers" )
 
         # TODO: rewrite this function to take lists of ips and ports?
@@ -244,16 +295,19 @@
                                     "not in the list of controllers s" +
                                     str( i ) + " is connecting to." )
                     mastershipCheck = main.FALSE
-        if mastershipCheck == main.TRUE:
-            main.log.report( "Switch mastership assigned correctly" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=mastershipCheck,
             onpass="Switch mastership assigned correctly",
             onfail="Switches not assigned correctly to controllers" )
+        # FIXME: Check topo here
+
+        main.step( "Assign mastership of switches to specific controllers" )
         # Manually assign mastership to the controller we want
         roleCall = main.TRUE
-        roleCheck = main.TRUE
+
+        ipList = [ ]
+        deviceList = []
         try:
             for i in range( 1, 29 ):  # switches 1 through 28
                 # set up correct variables:
@@ -297,16 +351,8 @@
                 # TODO: make this controller dynamic
                 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
                                                                   ip )
-                # Check assignment
-                master =  main.ONOScli1.getRole( deviceId ).get( 'master' )
-                if ip in master:
-                    roleCheck = roleCheck and main.TRUE
-                else:
-                    roleCheck = roleCheck and main.FALSE
-                    main.log.error( "Error, controller " + ip + " is not" +
-                                    " master " + "of device " +
-                                    str( deviceId ) + ". Master is " +
-                                    repr( master ) + "." )
+                ipList.append( ip )
+                deviceList.append( deviceId )
         except ( AttributeError, AssertionError ):
             main.log.exception( "Something is wrong with ONOS device view" )
             main.log.info( main.ONOScli1.devices() )
@@ -316,6 +362,24 @@
             onpass="Re-assigned switch mastership to designated controller",
             onfail="Something wrong with deviceRole calls" )
 
+        main.step( "Check mastership was correctly assigned" )
+        roleCheck = main.TRUE
+        # NOTE: This is due to the fact that device mastership change is not
+        #       atomic and is actually a multi step process
+        time.sleep( 5 )
+        for i in range( len( ipList ) ):
+            ip = ipList[i]
+            deviceId = deviceList[i]
+            # Check assignment
+            master = main.ONOScli1.getRole( deviceId ).get( 'master' )
+            if ip in master:
+                roleCheck = roleCheck and main.TRUE
+            else:
+                roleCheck = roleCheck and main.FALSE
+                main.log.error( "Error, controller " + ip + " is not" +
+                                " master " + "of device " +
+                                str( deviceId ) + ". Master is " +
+                                repr( master ) + "." )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=roleCheck,
@@ -339,17 +403,21 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        main.log.report( "Adding host intents" )
         main.case( "Adding host Intents" )
-
-        main.step( "Discovering  Hosts( Via pingall for now )" )
-        # FIXME: Once we have a host discovery mechanism, use that instead
+        main.caseExplaination = "Discover hosts by using pingall then " +\
+                                "assign predetermined host-to-host intents." +\
+                                " After installation, check that the intent" +\
+                                " is distributed to all nodes and the state" +\
+                                " is INSTALLED"
 
         # install onos-app-fwd
-        main.log.info( "Install reactive forwarding app" )
-        appResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+        main.step( "Install reactive forwarding app" )
+        installResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+        utilities.assert_equals( expect=main.TRUE, actual=installResults,
+                                 onpass="Install fwd successful",
+                                 onfail="Install fwd failed" )
 
-        # FIXME: add this to asserts
+        main.step( "Check app ids" )
         appCheck = main.TRUE
         threads = []
         for i in range( numControllers ):
@@ -362,13 +430,15 @@
         for t in threads:
             t.join()
             appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
         if appCheck != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
+        main.step( "Discovering Hosts( Via pingall for now )" )
+        # FIXME: Once we have a host discovery mechanism, use that instead
         # REACTIVE FWD test
         pingResult = main.FALSE
         for i in range(2):  # Retry if pingall fails first time
@@ -378,14 +448,28 @@
                 expect=main.TRUE,
                 actual=pingResult,
                 onpass="Reactive Pingall test passed",
-                onfail="Reactive Pingall failed, one or more ping pairs failed" )
+                onfail="Reactive Pingall failed, " +
+                       "one or more ping pairs failed" )
             time2 = time.time()
-            main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
-
+            main.log.info( "Time for pingall: %2f seconds" %
+                           ( time2 - time1 ) )
+        # timeout for fwd flows
+        time.sleep( 11 )
         # uninstall onos-app-fwd
-        main.log.info( "Uninstall reactive forwarding app" )
-        appResults = appResults and CLIs[0].deactivateApp( "org.onosproject.fwd" )
+        main.step( "Uninstall reactive forwarding app" )
+        uninstallResult = CLIs[0].deactivateApp( "org.onosproject.fwd" )
+        utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
+                                 onpass="Uninstall fwd successful",
+                                 onfail="Uninstall fwd failed" )
+        '''
+        main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ")
+        import time
+        time.sleep(60)
+        '''
+
+        main.step( "Check app ids" )
         threads = []
+        appCheck2 = main.TRUE
         for i in range( numControllers ):
             t = main.Thread( target=CLIs[i].appToIDCheck,
                              name="appToIDCheck-" + str( i ),
@@ -395,18 +479,15 @@
 
         for t in threads:
             t.join()
-            appCheck = appCheck and t.result
-        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
-                                 onpass="App Ids seem to be correct",
-                                 onfail="Something is wrong with app Ids" )
-        if appCheck != main.TRUE:
+            appCheck2 = appCheck2 and t.result
+        if appCheck2 != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
-        # timeout for fwd flows
-        time.sleep( 11 )
-
-        main.step( "Add host intents" )
+        main.step( "Add host intents via cli" )
         intentIds = []
         # TODO:  move the host numbers to params
         #        Maybe look at all the paths we ping?
@@ -449,7 +530,10 @@
                 except ( ValueError, TypeError ):
                     main.log.warn( repr( hosts ) )
                 hostResult = main.FALSE
-        # FIXME: DEBUG
+        utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+                                 onpass="Found a host id for each host",
+                                 onfail="Error looking up host ids" )
+
         intentStart = time.time()
         onosIds = main.ONOScli1.getAllIntentsId()
         main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -544,35 +628,49 @@
             main.log.exception( "Error parsing pending map" )
             main.log.error( repr( pendingMap ) )
 
-        intentAddResult = bool( pingResult and hostResult and intentAddResult
-                                and not missingIntents and installedCheck )
-        utilities.assert_equals(
-            expect=True,
-            actual=intentAddResult,
-            onpass="Pushed host intents to ONOS",
-            onfail="Error in pushing host intents to ONOS" )
-        for i in range(100):
-            onosIds = main.ONOScli1.getAllIntentsId()
+        intentAddResult = bool( intentAddResult and not missingIntents and
+                                installedCheck )
+        if not intentAddResult:
+            main.log.error( "Error in pushing host intents to ONOS" )
+
+        main.step( "Intent Anti-Entropy dispersion" )
+        for i in range(80):
+            correct = True
             main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
-            main.log.info( "Intents in ONOS: " + str( sorted( onosIds ) ) )
-            if sorted(onosIds) == sorted(intentIds):
+            for cli in CLIs:
+                onosIds = []
+                ids = cli.getAllIntentsId()
+                onosIds.append( ids )
+                main.log.debug( "Intents in " + cli.name + ": " +
+                                str( sorted( onosIds ) ) )
+                if sorted( ids ) != sorted( intentIds ):
+                    main.log.debug( "Set of intent IDs doesn't match" )
+                    correct = False
+                    break
+                else:
+                    intents = json.loads( cli.intents() )
+                    for intent in intents:
+                        if intent[ 'state' ] != "INSTALLED":
+                            main.log.warn( "Intent " + intent[ 'id' ] +
+                                           " is " + intent[ 'state' ] )
+                            correct = False
+                            break
+            if correct:
                 break
-            else:
-                time.sleep(1)
-        # FIXME: DEBUG
         if not intentStop:
             intentStop = time.time()
+        global gossipTime
         gossipTime = intentStop - intentStart
         main.log.info( "It took about " + str( gossipTime ) +
-                        " seconds for all intents to appear on ONOS1" )
+                        " seconds for all intents to appear in each node" )
         # FIXME: make this time configurable/calculate based off of number of
         #        nodes and gossip rounds
         utilities.assert_greater_equals(
-                expect=30, actual=gossipTime,
+                expect=40, actual=gossipTime,
                 onpass="ECM anti-entropy for intents worked within " +
                        "expected time",
                 onfail="Intent ECM anti-entropy took too long" )
-        if gossipTime <= 30:
+        if gossipTime <= 40:
             intentAddResult = True
 
         if not intentAddResult or "key" in pendingMap:
@@ -675,9 +773,11 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        description = " Ping across added host intents"
-        main.log.report( description )
-        main.case( description )
+        main.case( "Verify connectivity by sendind traffic across Intents" )
+        main.caseExplaination = "Ping across added host intents to check " +\
+                                "functionality and check the state of " +\
+                                "the intent"
+        main.step( "Ping across added host intents" )
         PingResult = main.TRUE
         for i in range( 8, 18 ):
             ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -690,7 +790,7 @@
                 main.log.info( "Ping test passed!" )
                 # Don't set PingResult or you'd override failures
         if PingResult == main.FALSE:
-            main.log.report(
+            main.log.error(
                 "Intents have not been installed correctly, pings failed." )
             # TODO: pretty print
             main.log.warn( "ONOS1 intents: " )
@@ -702,17 +802,17 @@
                                            separators=( ',', ': ' ) ) )
             except ( ValueError, TypeError ):
                 main.log.warn( repr( tmpIntents ) )
-        if PingResult == main.TRUE:
-            main.log.report(
-                "Intents have been installed correctly and verified by pings" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=PingResult,
             onpass="Intents have been installed correctly and pings work",
             onfail="Intents have not been installed correctly, pings failed." )
 
-        installedCheck = True
-        if PingResult is not main.TRUE:
+        main.step( "Check Intent state" )
+        installedCheck = False
+        count = 0
+        while not installedCheck and count < 40:
+            installedCheck = True
             # Print the intent states
             intents = main.ONOScli1.intents()
             intentStates = []
@@ -728,66 +828,88 @@
                     intentStates.append( ( intentId, state ) )
             except ( ValueError, TypeError ):
                 main.log.exception( "Error parsing intents." )
+            # Print states
             intentStates.sort()
             for i, s in intentStates:
                 count += 1
                 main.log.info( "%-6s%-15s%-15s" %
                                ( str( count ), str( i ), str( s ) ) )
-            leaders = main.ONOScli1.leaders()
-            try:
-                if leaders:
-                    parsedLeaders = json.loads( leaders )
-                    main.log.warn( json.dumps( parsedLeaders,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # check for all intent partitions
-                    # check for election
-                    topics = []
-                    for i in range( 14 ):
-                        topics.append( "intent-partition-" + str( i ) )
-                    # FIXME: this should only be after we start the app
-                    topics.append( "org.onosproject.election" )
-                    main.log.debug( topics )
-                    ONOStopics = [ j['topic'] for j in parsedLeaders ]
-                    for topic in topics:
-                        if topic not in ONOStopics:
-                            main.log.error( "Error: " + topic +
-                                            " not in leaders" )
-                else:
-                    main.log.error( "leaders() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing leaders" )
-                main.log.error( repr( leaders ) )
-            partitions = main.ONOScli1.partitions()
-            try:
-                if partitions :
-                    parsedPartitions = json.loads( partitions )
-                    main.log.warn( json.dumps( parsedPartitions,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # TODO check for a leader in all paritions
-                    # TODO check for consistency among nodes
-                else:
-                    main.log.error( "partitions() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing partitions" )
-                main.log.error( repr( partitions ) )
-            pendingMap = main.ONOScli1.pendingMap()
-            try:
-                if pendingMap :
-                    parsedPending = json.loads( pendingMap )
-                    main.log.warn( json.dumps( parsedPending,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # TODO check something here?
-                else:
-                    main.log.error( "pendingMap() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing pending map" )
-                main.log.error( repr( pendingMap ) )
+            if not installedCheck:
+                time.sleep( 1 )
+                count += 1
+        utilities.assert_equals( expect=True, actual=installedCheck,
+                                 onpass="Intents are all INSTALLED",
+                                 onfail="Intents are not all in " +
+                                        "INSTALLED state" )
+
+        main.step( "Check leadership of topics" )
+        leaders = main.ONOScli1.leaders()
+        topicCheck = main.TRUE
+        try:
+            if leaders:
+                parsedLeaders = json.loads( leaders )
+                main.log.warn( json.dumps( parsedLeaders,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # check for all intent partitions
+                # check for election
+                # TODO: Look at Devices as topics now that it uses this system
+                topics = []
+                for i in range( 14 ):
+                    topics.append( "intent-partition-" + str( i ) )
+                # FIXME: this should only be after we start the app
+                # FIXME: topics.append( "org.onosproject.election" )
+                # Print leaders output
+                main.log.debug( topics )
+                ONOStopics = [ j['topic'] for j in parsedLeaders ]
+                for topic in topics:
+                    if topic not in ONOStopics:
+                        main.log.error( "Error: " + topic +
+                                        " not in leaders" )
+                        topicCheck = main.FALSE
+            else:
+                main.log.error( "leaders() returned None" )
+                topicCheck = main.FALSE
+        except ( ValueError, TypeError ):
+            topicCheck = main.FALSE
+            main.log.exception( "Error parsing leaders" )
+            main.log.error( repr( leaders ) )
+            # TODO: Check for a leader of these topics
+        utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+                                 onpass="intent Partitions is in leaders",
+                                 onfail="Some topics were lost " )
+        # Print partitions
+        partitions = main.ONOScli1.partitions()
+        try:
+            if partitions :
+                parsedPartitions = json.loads( partitions )
+                main.log.warn( json.dumps( parsedPartitions,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # TODO check for a leader in all paritions
+                # TODO check for consistency among nodes
+            else:
+                main.log.error( "partitions() returned None" )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing partitions" )
+            main.log.error( repr( partitions ) )
+        # Print Pending Map
+        pendingMap = main.ONOScli1.pendingMap()
+        try:
+            if pendingMap :
+                parsedPending = json.loads( pendingMap )
+                main.log.warn( json.dumps( parsedPending,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # TODO check something here?
+            else:
+                main.log.error( "pendingMap() returned None" )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing pending map" )
+            main.log.error( repr( pendingMap ) )
 
         if not installedCheck:
             main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -868,6 +990,39 @@
             except ( ValueError, TypeError ):
                 main.log.exception( "Error parsing pending map" )
                 main.log.error( repr( pendingMap ) )
+        # Print flowrules
+        main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+        main.step( "Wait a minute then ping again" )
+        # the wait is above
+        PingResult = main.TRUE
+        for i in range( 8, 18 ):
+            ping = main.Mininet1.pingHost( src="h" + str( i ),
+                                           target="h" + str( i + 10 ) )
+            PingResult = PingResult and ping
+            if ping == main.FALSE:
+                main.log.warn( "Ping failed between h" + str( i ) +
+                               " and h" + str( i + 10 ) )
+            elif ping == main.TRUE:
+                main.log.info( "Ping test passed!" )
+                # Don't set PingResult or you'd override failures
+        if PingResult == main.FALSE:
+            main.log.error(
+                "Intents have not been installed correctly, pings failed." )
+            # TODO: pretty print
+            main.log.warn( "ONOS1 intents: " )
+            try:
+                tmpIntents = main.ONOScli1.intents()
+                main.log.warn( json.dumps( json.loads( tmpIntents ),
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+            except ( ValueError, TypeError ):
+                main.log.warn( repr( tmpIntents ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=PingResult,
+            onpass="Intents have been installed correctly and pings work",
+            onfail="Intents have not been installed correctly, pings failed." )
 
     def CASE5( self, main ):
         """
@@ -883,7 +1038,6 @@
         # assumes that sts is already in you PYTHONPATH
         from sts.topology.teston_topology import TestONTopology
 
-        main.log.report( "Setting up and gathering data for current state" )
         main.case( "Setting up and gathering data for current state" )
         # The general idea for this test case is to pull the state of
         # ( intents,flows, topology,... ) from each ONOS node
@@ -931,7 +1085,7 @@
 
         for i in range( numControllers ):
             if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " roles" )
                 main.log.warn(
                     "ONOS" + str( i + 1 ) + " mastership response: " +
@@ -945,7 +1099,7 @@
 
         main.step( "Check for consistency in roles from each controller" )
         if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
-            main.log.report(
+            main.log.info(
                 "Switch roles are consistent across all ONOS nodes" )
         else:
             consistentMastership = False
@@ -993,7 +1147,7 @@
 
         for i in range( numControllers ):
             if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " intents" )
                 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
                                repr( ONOSIntents[ i ] ) )
@@ -1006,34 +1160,68 @@
 
         main.step( "Check for consistency in Intents from each controller" )
         if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
-            main.log.report( "Intents are consistent across all ONOS " +
+            main.log.info( "Intents are consistent across all ONOS " +
                              "nodes" )
         else:
             consistentIntents = False
-            main.log.report( "Intents not consistent" )
+            main.log.error( "Intents not consistent" )
         utilities.assert_equals(
             expect=True,
             actual=consistentIntents,
             onpass="Intents are consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of intents" )
 
+        if intentsResults:
+            # Try to make it easy to figure out what is happening
+            #
+            # Intent      ONOS1      ONOS2    ...
+            #  0x01     INSTALLED  INSTALLING
+            #  ...        ...         ...
+            #  ...        ...         ...
+            title = "   Id"
+            for n in range( numControllers ):
+                title += " " * 10 + "ONOS" + str( n + 1 )
+            main.log.warn( title )
+            keys = []
+            try:
+                # Get the set of all intent keys
+                for nodeStr in ONOSIntents:
+                    node = json.loads( nodeStr )
+                    for intent in node:
+                        keys.append( intent.get( 'id' ) )
+                keys = set( keys )
+                # For each intent key, print the state on each node
+                for key in keys:
+                    row = "%-13s" % key
+                    for nodeStr in ONOSIntents:
+                        node = json.loads( nodeStr )
+                        for intent in node:
+                            if intent.get( 'id', "Error" ) == key:
+                                row += "%-15s" % intent.get( 'state' )
+                    main.log.warn( row )
+                # End of intent state table
+            except ValueError as e:
+                main.log.exception( e )
+                main.log.debug( "nodeStr was: " + repr( nodeStr ) )
+
         if intentsResults and not consistentIntents:
+            # print the json objects
             n = len(ONOSIntents)
-            main.log.warn( "ONOS" + str( n ) + " intents: " )
-            main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
-                                       sort_keys=True,
-                                       indent=4,
-                                       separators=( ',', ': ' ) ) )
+            main.log.debug( "ONOS" + str( n ) + " intents: " )
+            main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+                                        sort_keys=True,
+                                        indent=4,
+                                        separators=( ',', ': ' ) ) )
             for i in range( numControllers ):
                 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
-                    main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
-                    main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
+                    main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
+                    main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
+                                                sort_keys=True,
+                                                indent=4,
+                                                separators=( ',', ': ' ) ) )
                 else:
-                    main.log.warn( nodes[ i ].name + " intents match ONOS" +
-                                   str( n ) + " intents" )
+                    main.log.debug( nodes[ i ].name + " intents match ONOS" +
+                                    str( n ) + " intents" )
         elif intentsResults and consistentIntents:
             intentCheck = main.TRUE
             intentState = ONOSIntents[ 0 ]
@@ -1065,7 +1253,7 @@
         for i in range( numControllers ):
             num = str( i + 1 )
             if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
-                main.log.report( "Error in getting ONOS" + num + " flows" )
+                main.log.error( "Error in getting ONOS" + num + " flows" )
                 main.log.warn( "ONOS" + num + " flows response: " +
                                repr( ONOSFlows[ i ] ) )
                 flowsResults = False
@@ -1089,7 +1277,7 @@
         main.step( "Check for consistency in Flows from each controller" )
         tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
         if all( tmp ):
-            main.log.report( "Flow count is consistent across all ONOS nodes" )
+            main.log.info( "Flow count is consistent across all ONOS nodes" )
         else:
             consistentFlows = False
         utilities.assert_equals(
@@ -1244,6 +1432,7 @@
         # Compare json objects for hosts and dataplane clusters
 
         # hosts
+        main.step( "Host view is consistent across ONOS nodes" )
         consistentHostsResult = main.TRUE
         for controller in range( len( hosts ) ):
             controllerStr = str( controller + 1 )
@@ -1251,14 +1440,14 @@
                 if hosts[ controller ] == hosts[ 0 ]:
                     continue
                 else:  # hosts not consistent
-                    main.log.report( "hosts from ONOS" +
+                    main.log.error( "hosts from ONOS" +
                                      controllerStr +
                                      " is inconsistent with ONOS1" )
                     main.log.warn( repr( hosts[ controller ] ) )
                     consistentHostsResult = main.FALSE
 
             else:
-                main.log.report( "Error in getting ONOS hosts from ONOS" +
+                main.log.error( "Error in getting ONOS hosts from ONOS" +
                                  controllerStr )
                 consistentHostsResult = main.FALSE
                 main.log.warn( "ONOS" + controllerStr +
@@ -1270,11 +1459,12 @@
             onpass="Hosts view is consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of hosts" )
 
+        main.step( "Each host has an IP address" )
         ipResult = main.TRUE
         for controller in range( 0, len( hosts ) ):
             controllerStr = str( controller + 1 )
             for host in hosts[ controller ]:
-                if not host.get( 'ips', [ ] ):
+                if not host.get( 'ipAddresses', [ ] ):
                     main.log.error( "DEBUG:Error with host ips on controller" +
                                     controllerStr + ": " + str( host ) )
                     ipResult = main.FALSE
@@ -1285,6 +1475,7 @@
             onfail="The ip of at least one host is missing" )
 
         # Strongly connected clusters of devices
+        main.step( "Cluster view is consistent across ONOS nodes" )
         consistentClustersResult = main.TRUE
         for controller in range( len( clusters ) ):
             controllerStr = str( controller + 1 )
@@ -1292,12 +1483,12 @@
                 if clusters[ controller ] == clusters[ 0 ]:
                     continue
                 else:  # clusters not consistent
-                    main.log.report( "clusters from ONOS" + controllerStr +
+                    main.log.error( "clusters from ONOS" + controllerStr +
                                      " is inconsistent with ONOS1" )
                     consistentClustersResult = main.FALSE
 
             else:
-                main.log.report( "Error in getting dataplane clusters " +
+                main.log.error( "Error in getting dataplane clusters " +
                                  "from ONOS" + controllerStr )
                 consistentClustersResult = main.FALSE
                 main.log.warn( "ONOS" + controllerStr +
@@ -1309,6 +1500,7 @@
             onpass="Clusters view is consistent across all ONOS nodes",
             onfail="ONOS nodes have different views of clusters" )
         # there should always only be one cluster
+        main.step( "Cluster view correct across ONOS nodes" )
         try:
             numClusters = len( json.loads( clusters[ 0 ] ) )
         except ( ValueError, TypeError ):
@@ -1379,14 +1571,6 @@
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
 
-        finalAssert = main.TRUE
-        finalAssert = ( finalAssert and topoResult and flowCheck
-                        and intentCheck and consistentMastership
-                        and mastershipCheck and rolesNotNull )
-        utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
-                                 onpass="State check successful",
-                                 onfail="State check NOT successful" )
-
     def CASE6( self, main ):
         """
         The Failure case. Since this is the Sanity test, we do nothing.
@@ -1397,7 +1581,7 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
-        main.log.report( "Wait 60 seconds instead of inducing a failure" )
+        main.case( "Wait 60 seconds instead of inducing a failure" )
         time.sleep( 60 )
         utilities.assert_equals(
             expect=main.TRUE,
@@ -1437,6 +1621,7 @@
             onpass="Each device has a master",
             onfail="Some devices don't have a master assigned" )
 
+        main.step( "Read device roles from ONOS" )
         ONOSMastership = []
         mastershipCheck = main.FALSE
         consistentMastership = True
@@ -1455,7 +1640,7 @@
 
         for i in range( numControllers ):
             if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " roles" )
                 main.log.warn(
                     "ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1469,7 +1654,7 @@
 
         main.step( "Check for consistency in roles from each controller" )
         if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
-            main.log.report(
+            main.log.info(
                 "Switch roles are consistent across all ONOS nodes" )
         else:
             consistentMastership = False
@@ -1516,8 +1701,6 @@
             else:
                 main.log.warn( "Mastership of switch %s changed" % switchDPID )
                 mastershipCheck = main.FALSE
-        if mastershipCheck == main.TRUE:
-            main.log.report( "Mastership of Switches was not changed" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=mastershipCheck,
@@ -1545,7 +1728,7 @@
 
         for i in range( numControllers ):
             if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
-                main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+                main.log.error( "Error in getting ONOS" + str( i + 1 ) +
                                  " intents" )
                 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
                                repr( ONOSIntents[ i ] ) )
@@ -1558,10 +1741,38 @@
 
         main.step( "Check for consistency in Intents from each controller" )
         if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
-            main.log.report( "Intents are consistent across all ONOS " +
+            main.log.info( "Intents are consistent across all ONOS " +
                              "nodes" )
         else:
             consistentIntents = False
+
+        # Try to make it easy to figure out what is happening
+        #
+        # Intent      ONOS1      ONOS2    ...
+        #  0x01     INSTALLED  INSTALLING
+        #  ...        ...         ...
+        #  ...        ...         ...
+        title = "   ID"
+        for n in range( numControllers ):
+            title += " " * 10 + "ONOS" + str( n + 1 )
+        main.log.warn( title )
+        # get all intent keys in the cluster
+        keys = []
+        for nodeStr in ONOSIntents:
+            node = json.loads( nodeStr )
+            for intent in node:
+                keys.append( intent.get( 'id' ) )
+        keys = set( keys )
+        for key in keys:
+            row = "%-13s" % key
+            for nodeStr in ONOSIntents:
+                node = json.loads( nodeStr )
+                for intent in node:
+                    if intent.get( 'id' ) == key:
+                        row += "%-15s" % intent.get( 'state' )
+            main.log.warn( row )
+        # End table view
+
         utilities.assert_equals(
             expect=True,
             actual=consistentIntents,
@@ -1597,22 +1808,41 @@
         main.step( "Compare current intents with intents before the failure" )
         # NOTE: this requires case 5 to pass for intentState to be set.
         #      maybe we should stop the test if that fails?
-        sameIntents = main.TRUE
+        sameIntents = main.FALSE
         if intentState and intentState == ONOSIntents[ 0 ]:
             sameIntents = main.TRUE
-            main.log.report( "Intents are consistent with before failure" )
+            main.log.info( "Intents are consistent with before failure" )
         # TODO: possibly the states have changed? we may need to figure out
         #       what the acceptable states are
-        else:
+        elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+            sameIntents = main.TRUE
             try:
-                main.log.warn( "ONOS intents: " )
-                main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
-                                           sort_keys=True, indent=4,
-                                           separators=( ',', ': ' ) ) )
+                before = json.loads( intentState )
+                after = json.loads( ONOSIntents[ 0 ] )
+                for intent in before:
+                    if intent not in after:
+                        sameIntents = main.FALSE
+                        main.log.debug( "Intent is not currently in ONOS " +\
+                                        "(at least in the same form):" )
+                        main.log.debug( json.dumps( intent ) )
             except ( ValueError, TypeError ):
                 main.log.exception( "Exception printing intents" )
-                main.log.warn( repr( ONOSIntents[0] ) )
-            sameIntents = main.FALSE
+                main.log.debug( repr( ONOSIntents[0] ) )
+                main.log.debug( repr( intentState ) )
+        if sameIntents == main.FALSE:
+            try:
+                main.log.debug( "ONOS intents before: " )
+                main.log.debug( json.dumps( json.loads( intentState ),
+                                            sort_keys=True, indent=4,
+                                            separators=( ',', ': ' ) ) )
+                main.log.debug( "Current ONOS intents: " )
+                main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+                                            sort_keys=True, indent=4,
+                                            separators=( ',', ': ' ) ) )
+            except ( ValueError, TypeError ):
+                main.log.exception( "Exception printing intents" )
+                main.log.debug( repr( ONOSIntents[0] ) )
+                main.log.debug( repr( intentState ) )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=sameIntents,
@@ -1635,8 +1865,6 @@
             if FlowTables == main.FALSE:
                 main.log.info( "Differences in flow table for switch: s" +
                                str( i + 1 ) )
-        if FlowTables == main.TRUE:
-            main.log.report( "No changes were found in the flow tables" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=FlowTables,
@@ -1662,13 +1890,14 @@
             main.log.info( "There are multiple mininet process running" )
         elif LossInPings == main.FALSE:
             main.log.info( "No Loss in the pings" )
-            main.log.report( "No loss of dataplane connectivity" )
+            main.log.info( "No loss of dataplane connectivity" )
         utilities.assert_equals(
             expect=main.FALSE,
             actual=LossInPings,
             onpass="No Loss of connectivity",
             onfail="Loss of dataplane connectivity detected" )
 
+        main.step( "Leadership Election is still functional" )
         # Test of LeadershipElection
         # NOTE: this only works for the sanity test. In case of failures,
         #       leader will likely change
@@ -1684,34 +1913,21 @@
                 pass
             elif leaderN == main.FALSE:
                 # error in response
-                main.log.report( "Something is wrong with " +
+                main.log.error( "Something is wrong with " +
                                  "electionTestLeader function, check the" +
                                  " error logs" )
                 leaderResult = main.FALSE
             elif leader != leaderN:
                 leaderResult = main.FALSE
-                main.log.report( cli.name + " sees " + str( leaderN ) +
+                main.log.error( cli.name + " sees " + str( leaderN ) +
                                  " as the leader of the election app. " +
                                  "Leader should be " + str( leader ) )
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a new " +
-                             "leader was re-elected if applicable )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
             onpass="Leadership election passed",
             onfail="Something went wrong with Leadership election" )
 
-        result = ( mastershipCheck and intentCheck and FlowTables and
-                   ( not LossInPings ) and rolesNotNull and leaderResult )
-        result = int( result )
-        if result == main.TRUE:
-            main.log.report( "Constant State Tests Passed" )
-        utilities.assert_equals( expect=main.TRUE, actual=result,
-                                 onpass="Constant State Tests Passed",
-                                 onfail="Constant state tests failed" )
-
     def CASE8( self, main ):
         """
         Compare topo
@@ -1729,15 +1945,24 @@
         assert CLIs, "CLIs not defined"
         assert nodes, "nodes not defined"
 
-        description = "Compare ONOS Topology view to Mininet topology"
-        main.case( description )
-        main.log.report( description )
+        main.case( "Compare ONOS Topology view to Mininet topology" )
+        main.caseExplaination = "Compare topology objects between Mininet" +\
+                                " and ONOS"
         main.step( "Create TestONTopology object" )
-        ctrls = []
-        for node in nodes:
-            temp = ( node, node.name, node.ip_address, 6633 )
-            ctrls.append( temp )
-        MNTopo = TestONTopology( main.Mininet1, ctrls )
+        try:
+            ctrls = []
+            for node in nodes:
+                temp = ( node, node.name, node.ip_address, 6633 )
+                ctrls.append( temp )
+            MNTopo = TestONTopology( main.Mininet1, ctrls )
+        except Exception:
+            objResult = main.FALSE
+        else:
+            objResult = main.TRUE
+        utilities.assert_equals( expect=main.TRUE, actual=objResult,
+                                 onpass="Created TestONTopology object",
+                                 onfail="Exception while creating " +
+                                        "TestONTopology object" )
 
         main.step( "Comparing ONOS topology to MN" )
         devicesResults = main.TRUE
@@ -1788,9 +2013,9 @@
             for controller in range( 0, len( hosts ) ):
                 controllerStr = str( controller + 1 )
                 for host in hosts[ controller ]:
-                    if host is None or host.get( 'ips', [] ) == []:
+                    if host is None or host.get( 'ipAddresses', [] ) == []:
                         main.log.error(
-                            "DEBUG:Error with host ips on controller" +
+                            "DEBUG:Error with host ipAddresses on controller" +
                             controllerStr + ": " + str( host ) )
                         ipResult = main.FALSE
             ports = []
@@ -1896,6 +2121,7 @@
             # Compare json objects for hosts and dataplane clusters
 
             # hosts
+            main.step( "Hosts view is consistent across all ONOS nodes" )
             consistentHostsResult = main.TRUE
             for controller in range( len( hosts ) ):
                 controllerStr = str( controller + 1 )
@@ -1903,13 +2129,13 @@
                     if hosts[ controller ] == hosts[ 0 ]:
                         continue
                     else:  # hosts not consistent
-                        main.log.report( "hosts from ONOS" + controllerStr +
+                        main.log.error( "hosts from ONOS" + controllerStr +
                                          " is inconsistent with ONOS1" )
                         main.log.warn( repr( hosts[ controller ] ) )
                         consistentHostsResult = main.FALSE
 
                 else:
-                    main.log.report( "Error in getting ONOS hosts from ONOS" +
+                    main.log.error( "Error in getting ONOS hosts from ONOS" +
                                      controllerStr )
                     consistentHostsResult = main.FALSE
                     main.log.warn( "ONOS" + controllerStr +
@@ -1922,6 +2148,7 @@
                 onfail="ONOS nodes have different views of hosts" )
 
             # Strongly connected clusters of devices
+            main.step( "Clusters view is consistent across all ONOS nodes" )
             consistentClustersResult = main.TRUE
             for controller in range( len( clusters ) ):
                 controllerStr = str( controller + 1 )
@@ -1929,13 +2156,13 @@
                     if clusters[ controller ] == clusters[ 0 ]:
                         continue
                     else:  # clusters not consistent
-                        main.log.report( "clusters from ONOS" +
+                        main.log.error( "clusters from ONOS" +
                                          controllerStr +
                                          " is inconsistent with ONOS1" )
                         consistentClustersResult = main.FALSE
 
                 else:
-                    main.log.report( "Error in getting dataplane clusters " +
+                    main.log.error( "Error in getting dataplane clusters " +
                                      "from ONOS" + controllerStr )
                     consistentClustersResult = main.FALSE
                     main.log.warn( "ONOS" + controllerStr +
@@ -1947,6 +2174,8 @@
                 onpass="Clusters view is consistent across all ONOS nodes",
                 onfail="ONOS nodes have different views of clusters" )
             # there should always only be one cluster
+            main.step( "Topology view is correct and consistent across all " +
+                       "ONOS nodes" )
             try:
                 numClusters = len( json.loads( clusters[ 0 ] ) )
             except ( ValueError, TypeError ):
@@ -1977,12 +2206,11 @@
         utilities.assert_equals( expect=main.TRUE, actual=topoResult,
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
-        if topoResult == main.TRUE:
-            main.log.report( "ONOS topology view matches Mininet topology" )
 
         # FIXME: move this to an ONOS state case
         main.step( "Checking ONOS nodes" )
         nodesOutput = []
+        nodeResults = main.TRUE
         threads = []
         for i in range( numControllers ):
             t = main.Thread( target=CLIs[i].nodes,
@@ -1999,9 +2227,10 @@
             try:
                 current = json.loads( i )
                 for node in current:
+                    currentResult = main.FALSE
                     if node['ip'] in ips:  # node in nodes() output is in cell
                         if node['state'] == 'ACTIVE':
-                            pass  # as it should be
+                            currentResult = main.TRUE
                         else:
                             main.log.error( "Error in ONOS node availability" )
                             main.log.error(
@@ -2010,9 +2239,13 @@
                                                 indent=4,
                                                 separators=( ',', ': ' ) ) )
                             break
+                    nodeResults = nodeResults and currentResult
             except ( ValueError, TypeError ):
                 main.log.error( "Error parsing nodes output" )
                 main.log.warn( repr( i ) )
+        utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
 
     def CASE9( self, main ):
         """
@@ -2030,7 +2263,6 @@
 
         description = "Turn off a link to ensure that Link Discovery " +\
                       "is working properly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Kill Link between s3 and s28" )
@@ -2059,7 +2291,6 @@
 
         description = "Restore a link to ensure that Link Discovery is " + \
                       "working properly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Bring link between s3 and s28 back up" )
@@ -2087,14 +2318,13 @@
         switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
 
         description = "Killing a switch to ensure it is discovered correctly"
-        main.log.report( description )
         main.case( description )
         switch = main.params[ 'kill' ][ 'switch' ]
         switchDPID = main.params[ 'kill' ][ 'dpid' ]
 
         # TODO: Make this switch parameterizable
         main.step( "Kill " + switch )
-        main.log.report( "Deleting " + switch )
+        main.log.info( "Deleting " + switch )
         main.Mininet1.delSwitch( switch )
         main.log.info( "Waiting " + str( switchSleep ) +
                        " seconds for switch down to be discovered" )
@@ -2133,11 +2363,9 @@
         switchDPID = main.params[ 'kill' ][ 'dpid' ]
         links = main.params[ 'kill' ][ 'links' ].split()
         description = "Adding a switch to ensure it is discovered correctly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Add back " + switch )
-        main.log.report( "Adding back " + switch )
         main.Mininet1.addSwitch( switch, dpid=switchDPID )
         for peer in links:
             main.Mininet1.addLink( switch, peer )
@@ -2186,9 +2414,7 @@
         colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
                    'blue': '\033[94m', 'green': '\033[92m',
                    'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
-        description = "Test Cleanup"
-        main.log.report( description )
-        main.case( description )
+        main.case( "Test Cleanup" )
         main.step( "Killing tcpdumps" )
         main.Mininet2.stopTcpdump()
 
@@ -2235,7 +2461,10 @@
         time.sleep( 10 )
 
         main.step( "Stopping Mininet" )
-        main.Mininet1.stopNet()
+        mnResult = main.Mininet1.stopNet()
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet stopped",
+                                 onfail="MN cleanup NOT successful" )
 
         main.step( "Checking ONOS Logs for errors" )
         for node in nodes:
@@ -2246,15 +2475,70 @@
         main.step( "Packing and rotating pcap archives" )
         os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
 
-        # TODO: actually check something here
-        utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
-                                 onpass="Test cleanup successful",
-                                 onfail="Test cleanup NOT successful" )
+        try:
+            timerLog = open( main.logdir + "/Timers.csv", 'w')
+            # Overwrite with empty line and close
+            labels = "Gossip Intents"
+            data = str( gossipTime )
+            timerLog.write( labels + "\n" + data )
+            timerLog.close()
+        except NameError, e:
+            main.log.exception(e)
 
     def CASE14( self, main ):
         """
         start election app on all onos nodes
         """
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+
+        main.case("Start Leadership Election app")
+        main.step( "Install leadership election app" )
+        appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=appResult,
+            onpass="Election app installed",
+            onfail="Something went wrong with installing Leadership election" )
+
+        main.step( "Run for election on each node" )
+        leaderResult = main.TRUE
+        leaders = []
+        for cli in CLIs:
+            cli.electionTestRun()
+        for cli in CLIs:
+            leader = cli.electionTestLeader()
+            if leader is None or leader == main.FALSE:
+                main.log.error( cli.name + ": Leader for the election app " +
+                                 "should be an ONOS node, instead got '" +
+                                 str( leader ) + "'" )
+                leaderResult = main.FALSE
+            leaders.append( leader )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=leaderResult,
+            onpass="Successfully ran for leadership",
+            onfail="Failed to run for leadership" )
+
+        main.step( "Check that each node shows the same leader" )
+        sameLeader = main.TRUE
+        if len( set( leaders ) ) != 1:
+            sameLeader = main.FALSE
+            main.log.error( "Results of electionTestLeader is order of CLIs:" +
+                            str( leaders ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=sameLeader,
+            onpass="Leadership is consistent for the election topic",
+            onfail="Nodes have different leaders" )
+
+    def CASE15( self, main ):
+        """
+        Check that Leadership Election is still functional
+        """
         import time
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
@@ -2263,51 +2547,31 @@
         assert nodes, "nodes not defined"
 
         leaderResult = main.TRUE
-        main.log.info( "Install leadership election app" )
-        main.ONOScli1.activateApp( "org.onosproject.election" )
+        description = "Check that Leadership Election is still functional"
+        main.case( description )
+
+        main.step( "Check that each node shows the same leader" )
+        sameLeader = main.TRUE
         leaders = []
         for cli in CLIs:
             leader = cli.electionTestLeader()
-            if leader is None or leader == main.FALSE:
-                main.log.report( cli.name + ": Leader for the election app " +
-                                 "should be an ONOS node, instead got '" +
-                                 str( leader ) + "'" )
-                leaderResult = main.FALSE
             leaders.append( leader )
         if len( set( leaders ) ) != 1:
-            leaderResult = main.FALSE
+            sameLeader = main.FALSE
             main.log.error( "Results of electionTestLeader is order of CLIs:" +
                             str( leaders ) )
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a leader " +
-                             "was elected )" )
         utilities.assert_equals(
             expect=main.TRUE,
-            actual=leaderResult,
-            onpass="Leadership election passed",
-            onfail="Something went wrong with Leadership election" )
+            actual=sameLeader,
+            onpass="Leadership is consistent for the election topic",
+            onfail="Nodes have different leaders" )
 
-    def CASE15( self, main ):
-        """
-        Check that Leadership Election is still functional
-        """
-        assert numControllers, "numControllers not defined"
-        assert main, "main not defined"
-        assert utilities.assert_equals, "utilities.assert_equals not defined"
-        assert CLIs, "CLIs not defined"
-        assert nodes, "nodes not defined"
-
-        leaderResult = main.TRUE
-        description = "Check that Leadership Election is still functional"
-        main.log.report( description )
-        main.case( description )
         main.step( "Find current leader and withdraw" )
         leader = main.ONOScli1.electionTestLeader()
         # do some sanity checking on leader before using it
         withdrawResult = main.FALSE
         if leader is None or leader == main.FALSE:
-            main.log.report(
+            main.log.error(
                 "Leader for the election app should be an ONOS node," +
                 "instead got '" + str( leader ) + "'" )
             leaderResult = main.FALSE
@@ -2323,8 +2587,8 @@
         utilities.assert_equals(
             expect=main.TRUE,
             actual=withdrawResult,
-            onpass="App was withdrawn from election",
-            onfail="App was not withdrawn from election" )
+            onpass="Node was withdrawn from election",
+            onfail="Node was not withdrawn from election" )
 
         main.step( "Make sure new leader is elected" )
         # FIXME: use threads
@@ -2333,19 +2597,20 @@
             leaderN = cli.electionTestLeader()
             leaderList.append( leaderN )
             if leaderN == leader:
-                main.log.report(  cli.name + " still sees " + str( leader ) +
+                main.log.error(  cli.name + " still sees " + str( leader ) +
                                   " as leader after they withdrew" )
                 leaderResult = main.FALSE
             elif leaderN == main.FALSE:
                 # error in  response
                 # TODO: add check for "Command not found:" in the driver, this
                 #       means the app isn't loaded
-                main.log.report( "Something is wrong with " +
+                main.log.error( "Something is wrong with " +
                                  "electionTestLeader function, " +
                                  "check the error logs" )
                 leaderResult = main.FALSE
             elif leaderN is None:
                 # node may not have recieved the event yet
+                time.sleep(7)
                 leaderN = cli.electionTestLeader()
                 leaderList.pop()
                 leaderList.append( leaderN )
@@ -2356,17 +2621,12 @@
                            "' as the leader" )
             consistentLeader = main.TRUE
         else:
-            main.log.report(
+            main.log.error(
                 "Inconsistent responses for leader of Election-app:" )
             for n in range( len( leaderList ) ):
-                main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+                main.log.error( "ONOS" + str( n + 1 ) + " response: " +
                                  str( leaderList[ n ] ) )
         leaderResult = leaderResult and consistentLeader
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a new " +
-                             "leader was elected when the old leader " +
-                             "resigned )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
@@ -2384,18 +2644,1110 @@
             actual=runResult,
             onpass="App re-ran for election",
             onfail="App failed to run for election" )
-        if consistentLeader == main.TRUE:
-            afterRun = main.ONOScli1.electionTestLeader()
-            # verify leader didn't just change
-            if afterRun == leaderList[ 0 ]:
-                leaderResult = main.TRUE
-            else:
-                leaderResult = main.FALSE
-        # TODO: assert on  run and withdraw results?
+
+        main.step( "Leader did not change when old leader re-ran" )
+        afterRun = main.ONOScli1.electionTestLeader()
+        # verify leader didn't just change
+        if afterRun == leaderList[ 0 ]:
+            afterResult = main.TRUE
+        else:
+            afterResult = main.FALSE
 
         utilities.assert_equals(
             expect=main.TRUE,
-            actual=leaderResult,
-            onpass="Leadership election passed",
+            actual=afterResult,
+            onpass="Old leader successfully re-ran for election",
             onfail="Something went wrong with Leadership election after " +
                    "the old leader re-ran for election" )
+
+    def CASE16( self, main ):
+        """
+        Install Distributed Primitives app
+        """
+        import time
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+
+        # Variables for the distributed primitives tests
+        global pCounterName
+        global iCounterName
+        global pCounterValue
+        global iCounterValue
+        global onosSet
+        global onosSetName
+        pCounterName = "TestON-Partitions"
+        iCounterName = "TestON-inMemory"
+        pCounterValue = 0
+        iCounterValue = 0
+        onosSet = set([])
+        onosSetName = "TestON-set"
+
+        description = "Install Primitives app"
+        main.case( description )
+        main.step( "Install Primitives app" )
+        appName = "org.onosproject.distributedprimitives"
+        appResults = CLIs[0].activateApp( appName )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=appResults,
+                                 onpass="Primitives app activated",
+                                 onfail="Primitives app not activated" )
+        time.sleep( 5 )  # To allow all nodes to activate
+
+    def CASE17( self, main ):
+        """
+        Check for basic functionality with distributed primitives
+        """
+        # Make sure variables are defined/set
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+        assert pCounterName, "pCounterName not defined"
+        assert iCounterName, "iCounterName not defined"
+        assert onosSetName, "onosSetName not defined"
+        # NOTE: assert fails if value is 0/None/Empty/False
+        try:
+            pCounterValue
+        except NameError:
+            main.log.error( "pCounterValue not defined, setting to 0" )
+            pCounterValue = 0
+        try:
+            iCounterValue
+        except NameError:
+            main.log.error( "iCounterValue not defined, setting to 0" )
+            iCounterValue = 0
+        try:
+            onosSet
+        except NameError:
+            main.log.error( "onosSet not defined, setting to empty Set" )
+            onosSet = set([])
+        # Variables for the distributed primitives tests. These are local only
+        addValue = "a"
+        addAllValue = "a b c d e f"
+        retainValue = "c d e f"
+
+        description = "Check for basic functionality with distributed " +\
+                      "primitives"
+        main.case( description )
+        main.caseExplaination = "Test the methods of the distributed primitives (counters and sets) throught the cli"
+        # DISTRIBUTED ATOMIC COUNTERS
+        main.step( "Increment and get a default counter on each node" )
+        pCounters = []
+        threads = []
+        addedPValues = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counterTestIncrement,
+                             name="counterIncrement-" + str( i ),
+                             args=[ pCounterName ] )
+            pCounterValue += 1
+            addedPValues.append( pCounterValue )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            pCounters.append( t.result )
+        # Check that counter incremented numController times
+        pCounterResults = True
+        for i in addedPValues:
+            tmpResult = i in pCounters
+            pCounterResults = pCounterResults and tmpResult
+            if not tmpResult:
+                main.log.error( str( i ) + " is not in partitioned "
+                                "counter incremented results" )
+        utilities.assert_equals( expect=True,
+                                 actual=pCounterResults,
+                                 onpass="Default counter incremented",
+                                 onfail="Error incrementing default" +
+                                        " counter" )
+
+        main.step( "Increment and get an in memory counter on each node" )
+        iCounters = []
+        addedIValues = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counterTestIncrement,
+                             name="icounterIncrement-" + str( i ),
+                             args=[ iCounterName ],
+                             kwargs={ "inMemory": True } )
+            iCounterValue += 1
+            addedIValues.append( iCounterValue )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            iCounters.append( t.result )
+        # Check that counter incremented numController times
+        iCounterResults = True
+        for i in addedIValues:
+            tmpResult = i in iCounters
+            iCounterResults = iCounterResults and tmpResult
+            if not tmpResult:
+                main.log.error( str( i ) + " is not in the in-memory "
+                                "counter incremented results" )
+        utilities.assert_equals( expect=True,
+                                 actual=iCounterResults,
+                                 onpass="In memory counter incremented",
+                                 onfail="Error incrementing in memory" +
+                                        " counter" )
+
+        main.step( "Check counters are consistant across nodes" )
+        onosCounters = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counters,
+                             name="counters-" + str( i ) )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            onosCounters.append( t.result )
+        tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+        if all( tmp ):
+            main.log.info( "Counters are consistent across all nodes" )
+            consistentCounterResults = main.TRUE
+        else:
+            main.log.error( "Counters are not consistent across all nodes" )
+            consistentCounterResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=consistentCounterResults,
+                                 onpass="ONOS counters are consistent " +
+                                        "across nodes",
+                                 onfail="ONOS Counters are inconsistent " +
+                                        "across nodes" )
+
+        main.step( "Counters we added have the correct values" )
+        correctResults = main.TRUE
+        for i in range( numControllers ):
+            current = json.loads( onosCounters[i] )
+            pValue = None
+            iValue = None
+            try:
+                for database in current:
+                    partitioned = database.get( 'partitionedDatabaseCounters' )
+                    if partitioned:
+                        for value in partitioned:
+                            if value.get( 'name' ) == pCounterName:
+                                pValue = value.get( 'value' )
+                                break
+                    inMemory = database.get( 'inMemoryDatabaseCounters' )
+                    if inMemory:
+                        for value in inMemory:
+                            if value.get( 'name' ) == iCounterName:
+                                iValue = value.get( 'value' )
+                                break
+            except AttributeError, e:
+                main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
+                                "is not as expected" )
+                correctResults = main.FALSE
+            if pValue == pCounterValue:
+                main.log.info( "Partitioned counter value is correct" )
+            else:
+                main.log.error( "Partitioned counter value is incorrect," +
+                                " expected value: " + str( pCounterValue )
+                                + " current value: " + str( pValue ) )
+                correctResults = main.FALSE
+            if iValue == iCounterValue:
+                main.log.info( "In memory counter value is correct" )
+            else:
+                main.log.error( "In memory counter value is incorrect, " +
+                                "expected value: " + str( iCounterValue ) +
+                                " current value: " + str( iValue ) )
+                correctResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=correctResults,
+                                 onpass="Added counters are correct",
+                                 onfail="Added counters are incorrect" )
+        # DISTRIBUTED SETS
+        main.step( "Distributed Set get" )
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=getResults,
+                                 onpass="Set elements are correct",
+                                 onfail="Set elements are incorrect" )
+
+        main.step( "Distributed Set size" )
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=sizeResults,
+                                 onpass="Set sizes are correct",
+                                 onfail="Set sizes are incorrect" )
+
+        main.step( "Distributed Set add()" )
+        onosSet.add( addValue )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAdd-" + str( i ),
+                             args=[ onosSetName, addValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addResults = main.FALSE
+            else:
+                # unexpected result
+                addResults = main.FALSE
+        if addResults != main.TRUE:
+            main.log.error( "Error executing set add" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addResults = addResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addResults,
+                                 onpass="Set add correct",
+                                 onfail="Set add was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set contains()" )
+        containsResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setContains-" + str( i ),
+                             args=[ onosSetName ],
+                             kwargs={ "values": addValue } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            # NOTE: This is the tuple
+            containsResponses.append( t.result )
+
+        containsResults = main.TRUE
+        for i in range( numControllers ):
+            if containsResponses[ i ] == main.ERROR:
+                containsResults = main.FALSE
+            else:
+                containsResults = containsResults and\
+                                  containsResponses[ i ][ 1 ]
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=containsResults,
+                                 onpass="Set contains is functional",
+                                 onfail="Set contains failed" )
+
+        main.step( "Distributed Set containsAll()" )
+        containsAllResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setContainsAll-" + str( i ),
+                             args=[ onosSetName ],
+                             kwargs={ "values": addAllValue } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            # NOTE: This is the tuple
+            containsAllResponses.append( t.result )
+
+        containsAllResults = main.TRUE
+        for i in range( numControllers ):
+            if containsResponses[ i ] == main.ERROR:
+                containsResults = main.FALSE
+            else:
+                containsResults = containsResults and\
+                                  containsResponses[ i ][ 1 ]
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=containsAllResults,
+                                 onpass="Set containsAll is functional",
+                                 onfail="Set containsAll failed" )
+
+        main.step( "Distributed Set remove()" )
+        onosSet.remove( addValue )
+        removeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestRemove-" + str( i ),
+                             args=[ onosSetName, addValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            removeResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        removeResults = main.TRUE
+        for i in range( numControllers ):
+            if removeResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif removeResponses[ i ] == main.FALSE:
+                # not in set, probably fine
+                pass
+            elif removeResponses[ i ] == main.ERROR:
+                # Error in execution
+                removeResults = main.FALSE
+            else:
+                # unexpected result
+                removeResults = main.FALSE
+        if removeResults != main.TRUE:
+            main.log.error( "Error executing set remove" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        removeResults = removeResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=removeResults,
+                                 onpass="Set remove correct",
+                                 onfail="Set remove was incorrect" )
+
+        main.step( "Distributed Set removeAll()" )
+        onosSet.difference_update( addAllValue.split() )
+        removeAllResponses = []
+        threads = []
+        try:
+            for i in range( numControllers ):
+                t = main.Thread( target=CLIs[i].setTestRemove,
+                                 name="setTestRemoveAll-" + str( i ),
+                                 args=[ onosSetName, addAllValue ] )
+                threads.append( t )
+                t.start()
+            for t in threads:
+                t.join()
+                removeAllResponses.append( t.result )
+        except Exception, e:
+            main.log.exception(e)
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        removeAllResults = main.TRUE
+        for i in range( numControllers ):
+            if removeAllResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif removeAllResponses[ i ] == main.FALSE:
+                # not in set, probably fine
+                pass
+            elif removeAllResponses[ i ] == main.ERROR:
+                # Error in execution
+                removeAllResults = main.FALSE
+            else:
+                # unexpected result
+                removeAllResults = main.FALSE
+        if removeAllResults != main.TRUE:
+            main.log.error( "Error executing set removeAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        removeAllResults = removeAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=removeAllResults,
+                                 onpass="Set removeAll correct",
+                                 onfail="Set removeAll was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set clear()" )
+        onosSet.clear()
+        clearResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestClear-" + str( i ),
+                             args=[ onosSetName, " "],  # Values doesn't matter
+                             kwargs={ "clear": True } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            clearResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        clearResults = main.TRUE
+        for i in range( numControllers ):
+            if clearResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif clearResponses[ i ] == main.FALSE:
+                # Nothing set, probably fine
+                pass
+            elif clearResponses[ i ] == main.ERROR:
+                # Error in execution
+                clearResults = main.FALSE
+            else:
+                # unexpected result
+                clearResults = main.FALSE
+        if clearResults != main.TRUE:
+            main.log.error( "Error executing set clear" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        clearResults = clearResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=clearResults,
+                                 onpass="Set clear correct",
+                                 onfail="Set clear was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set retain()" )
+        onosSet.intersection_update( retainValue.split() )
+        retainResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestRetain-" + str( i ),
+                             args=[ onosSetName, retainValue ],
+                             kwargs={ "retain": True } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            retainResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        retainResults = main.TRUE
+        for i in range( numControllers ):
+            if retainResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif retainResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif retainResponses[ i ] == main.ERROR:
+                # Error in execution
+                retainResults = main.FALSE
+            else:
+                # unexpected result
+                retainResults = main.FALSE
+        if retainResults != main.TRUE:
+            main.log.error( "Error executing set retain" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " +
+                                str( size ) + " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        retainResults = retainResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=retainResults,
+                                 onpass="Set retain correct",
+                                 onfail="Set retain was incorrect" )
+
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
index d3eb8dd..108ab88 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
@@ -16,13 +16,14 @@
     #CASE15: Check that Leadership Election is still functional
     #1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13
     #extra hosts test 1,2,8,11,8,12,8
-    <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,3,4,5,14,15,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
     <ENV>
         <cellName>HA</cellName>
     </ENV>
     <Git>False</Git>
     <branch> master </branch>
     <num_controllers> 1 </num_controllers>
+    <tcpdump> False </tcpdump>
 
     <CTRL>
         <ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index d2f0bc9..8f3c604 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -18,6 +18,8 @@
 CASE13: Clean up
 CASE14: start election app on all onos nodes
 CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
 """
 
 
@@ -44,9 +46,12 @@
         start cli sessions
         start tcpdump
         """
-        main.log.report( "ONOS Single node cluster restart " +
+        main.log.info( "ONOS Single node cluster restart " +
                          "HA test - initialization" )
         main.case( "Setting up test environment" )
+        main.caseExplaination = "Setup the test environment including " +\
+                                "installing ONOS, starting Mininet and ONOS" +\
+                                "cli sessions."
         # TODO: save all the timers and output them for plotting
 
         # load some variables from the params file
@@ -90,69 +95,117 @@
         verifyResult = main.ONOSbench.verifyCell()
 
         # FIXME:this is short term fix
-        main.log.report( "Removing raft logs" )
+        main.log.info( "Removing raft logs" )
         main.ONOSbench.onosRemoveRaftLogs()
 
-        main.log.report( "Uninstalling ONOS" )
+        main.log.info( "Uninstalling ONOS" )
         for node in nodes:
             main.ONOSbench.onosUninstall( node.ip_address )
 
+        # Make sure ONOS is DEAD
+        main.log.info( "Killing any ONOS processes" )
+        killResults = main.TRUE
+        for node in nodes:
+            killed = main.ONOSbench.onosKill( node.ip_address )
+            killResults = killResults and killed
+
         cleanInstallResult = main.TRUE
         gitPullResult = main.TRUE
 
         main.step( "Starting Mininet" )
-        main.Mininet1.startNet( )
+        mnResult = main.Mininet1.startNet( )
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet Started",
+                                 onfail="Error starting Mininet" )
 
-        main.step( "Compiling the latest version of ONOS" )
+        main.step( "Git checkout and pull " + gitBranch )
         if PULLCODE:
-            main.step( "Git checkout and pull " + gitBranch )
             main.ONOSbench.gitCheckout( gitBranch )
             gitPullResult = main.ONOSbench.gitPull()
-            if gitPullResult == main.ERROR:
-                main.log.error( "Error pulling git branch" )
+            # values of 1 or 3 are good
+            utilities.assert_lesser( expect=0, actual=gitPullResult,
+                                      onpass="Git pull successful",
+                                      onfail="Git pull failed" )
+        main.ONOSbench.getVersion( report=True )
 
-            main.step( "Using mvn clean & install" )
+        main.step( "Using mvn clean install" )
+        cleanInstallResult = main.TRUE
+        if PULLCODE and gitPullResult == main.TRUE:
             cleanInstallResult = main.ONOSbench.cleanInstall()
         else:
             main.log.warn( "Did not pull new code so skipping mvn " +
                            "clean install" )
-        main.ONOSbench.getVersion( report=True )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=cleanInstallResult,
+                                 onpass="MCI successful",
+                                 onfail="MCI failed" )
+        # GRAPHS
+        # NOTE: important params here:
+        #       job = name of Jenkins job
+        #       Plot Name = Plot-HA, only can be used if multiple plots
+        #       index = The number of the graph under plot name
+        job = "HASingleInstanceRestart"
+        plotName = "Plot-HA"
+        graphs = '<ac:structured-macro ac:name="html">\n'
+        graphs += '<ac:plain-text-body><![CDATA[\n'
+        graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+                  '/plot/' + plotName + '/getPlot?index=0' +\
+                  '&width=500&height=300"' +\
+                  'noborder="0" width="500" height="300" scrolling="yes" ' +\
+                  'seamless="seamless"></iframe>\n'
+        graphs += ']]></ac:plain-text-body>\n'
+        graphs += '</ac:structured-macro>\n'
+        main.log.wiki(graphs)
 
         cellResult = main.ONOSbench.setCell( "SingleHA" )
         verifyResult = main.ONOSbench.verifyCell()
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()
+        utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+                                 onpass="ONOS package successful",
+                                 onfail="ONOS package failed" )
 
         main.step( "Installing ONOS package" )
-        onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
+        onosInstallResult = main.ONOSbench.onosInstall( options="-f",
                                                            node=ONOS1Ip )
+        utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+                                 onpass="ONOS install successful",
+                                 onfail="ONOS install failed" )
 
         main.step( "Checking if ONOS is up yet" )
         for i in range( 2 ):
             onos1Isup = main.ONOSbench.isup( ONOS1Ip )
             if onos1Isup:
                 break
-        if not onos1Isup:
-            main.log.report( "ONOS1 didn't start!" )
+        utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
+                                 onpass="ONOS startup successful",
+                                 onfail="ONOS startup failed" )
 
-        cliResult = main.ONOScli1.startOnosCli( ONOS1Ip )
+        main.log.step( "Starting ONOS CLI sessions" )
+        cliResults = main.ONOScli1.startOnosCli( ONOS1Ip )
+        utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                 onpass="ONOS cli startup successful",
+                                 onfail="ONOS cli startup failed" )
 
-        main.step( "Start Packet Capture MN" )
-        main.Mininet2.startTcpdump(
-            str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
-            + "-MN.pcap",
-            intf=main.params[ 'MNtcpdump' ][ 'intf' ],
-            port=main.params[ 'MNtcpdump' ][ 'port' ] )
+        if main.params[ 'tcpdump' ].lower() == "true":
+            main.step( "Start Packet Capture MN" )
+            main.Mininet2.startTcpdump(
+                str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+                + "-MN.pcap",
+                intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+                port=main.params[ 'MNtcpdump' ][ 'port' ] )
 
-        case1Result = ( cleanInstallResult and packageResult and
-                        cellResult and verifyResult and onos1InstallResult
-                        and onos1Isup and cliResult )
+        main.step( "App Ids check" )
+        appCheck = main.ONOScli1.appToIDCheck()
+        if appCheck != main.TRUE:
+            main.log.warn( CLIs[0].apps() )
+            main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
-        utilities.assert_equals( expect=main.TRUE, actual=case1Result,
-                                 onpass="Test startup successful",
-                                 onfail="Test startup NOT successful" )
-
-        if case1Result == main.FALSE:
+        if cliResults == main.FALSE:
+            main.log.error( "Failed to start ONOS, stopping test" )
             main.cleanup()
             main.exit()
 
@@ -172,8 +225,10 @@
         assert ONOS6Port, "ONOS6Port not defined"
         assert ONOS7Port, "ONOS7Port not defined"
 
-        main.log.report( "Assigning switches to controllers" )
         main.case( "Assigning Controllers" )
+        main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+                                "and check that an ONOS node becomes the " +\
+                                "master of the device."
         main.step( "Assign switches to controllers" )
 
         for i in range( 1, 29 ):
@@ -193,13 +248,14 @@
             else:
                 mastershipCheck = main.FALSE
         if mastershipCheck == main.TRUE:
-            main.log.report( "Switch mastership assigned correctly" )
+            main.log.info( "Switch mastership assigned correctly" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=mastershipCheck,
             onpass="Switch mastership assigned correctly",
             onfail="Switches not assigned correctly to controllers" )
 
+        main.step( "Assign mastership of switches to specific controllers" )
         roleCall = main.TRUE
         roleCheck = main.TRUE
         try:
@@ -237,7 +293,7 @@
                 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
                                                                   ip )
                 # Check assignment
-                master =  main.ONOScli1.getRole( deviceId ).get( 'master' )
+                master = main.ONOScli1.getRole( deviceId ).get( 'master' )
                 if ip in master:
                     roleCheck = roleCheck and main.TRUE
                 else:
@@ -255,6 +311,7 @@
             onpass="Re-assigned switch mastership to designated controller",
             onfail="Something wrong with deviceRole calls" )
 
+        main.step( "Check mastership was correctly assigned" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=roleCheck,
@@ -276,24 +333,33 @@
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
-        # FIXME: we must reinstall intents until we have a persistant
-        # datastore!
-        main.log.report( "Adding host intents" )
+        # NOTE: we must reinstall intents until we have a persistant intent
+        #        datastore!
         main.case( "Adding host Intents" )
-
-        main.step( "Discovering  Hosts( Via pingall for now )" )
-        # FIXME: Once we have a host discovery mechanism, use that instead
+        main.caseExplaination = "Discover hosts by using pingall then " +\
+                                "assign predetermined host-to-host intents." +\
+                                " After installation, check that the intent" +\
+                                " is distributed to all nodes and the state" +\
+                                " is INSTALLED"
 
         # install onos-app-fwd
-        main.log.info( "Install reactive forwarding app" )
-        appResults = main.ONOScli1.activateApp( "org.onosproject.fwd" )
+        main.step( "Install reactive forwarding app" )
+        installResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+        utilities.assert_equals( expect=main.TRUE, actual=installResults,
+                                 onpass="Install fwd successful",
+                                 onfail="Install fwd failed" )
 
-        # FIXME: add this to asserts
+        main.step( "Check app ids" )
         appCheck = main.ONOScli1.appToIDCheck()
         if appCheck != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
+        main.step( "Discovering Hosts( Via pingall for now )" )
+        # FIXME: Once we have a host discovery mechanism, use that instead
         # REACTIVE FWD test
         pingResult = main.FALSE
         for i in range(2):  # Retry if pingall fails first time
@@ -303,22 +369,30 @@
                 expect=main.TRUE,
                 actual=pingResult,
                 onpass="Reactive Pingall test passed",
-                onfail="Reactive Pingall failed, one or more ping pairs failed" )
+                onfail="Reactive Pingall failed, " +
+                       "one or more ping pairs failed" )
             time2 = time.time()
-            main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
-
+            main.log.info( "Time for pingall: %2f seconds" %
+                           ( time2 - time1 ) )
+        # timeout for fwd flows
+        time.sleep( 11 )
         # uninstall onos-app-fwd
-        main.log.info( "Uninstall reactive forwarding app" )
-        appResults = appResults and main.ONOScli1.deactivateApp( "org.onosproject.fwd" )
+        main.step( "Uninstall reactive forwarding app" )
+        uninstallResult = CLIs[0].deactivateApp( "org.onosproject.fwd" )
+        utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
+                                 onpass="Uninstall fwd successful",
+                                 onfail="Uninstall fwd failed" )
+
+        main.step( "Check app ids" )
         appCheck2 = main.ONOScli1.appToIDCheck()
         if appCheck2 != main.TRUE:
             main.log.warn( CLIs[0].apps() )
             main.log.warn( CLIs[0].appIDs() )
+        utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
+                                 onpass="App Ids seem to be correct",
+                                 onfail="Something is wrong with app Ids" )
 
-        # timeout for fwd flows
-        time.sleep( 11 )
-
-        main.step( "Add host intents" )
+        main.step( "Add host intents via cli" )
         intentIds = []
         # TODO:  move the host numbers to params
         #        Maybe look at all the paths we ping?
@@ -360,7 +434,10 @@
                 except ( ValueError, TypeError ):
                     main.log.warn( repr( hosts ) )
                 hostResult = main.FALSE
-        # FIXME: DEBUG
+        utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+                                 onpass="Found a host id for each host",
+                                 onfail="Error looking up host ids" )
+
         intentStart = time.time()
         onosIds = main.ONOScli1.getAllIntentsId()
         main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -455,13 +532,42 @@
             main.log.exception( "Error parsing pending map" )
             main.log.error( repr( pendingMap ) )
 
-        intentAddResult = bool( pingResult and hostResult and intentAddResult
-                                and not missingIntents and installedCheck )
-        utilities.assert_equals(
-            expect=True,
-            actual=intentAddResult,
-            onpass="Pushed host intents to ONOS",
-            onfail="Error in pushing host intents to ONOS" )
+        intentAddResult = bool( intentAddResult and not missingIntents and
+                                installedCheck )
+        if not intentAddResult:
+            main.log.error( "Error in pushing host intents to ONOS" )
+
+        main.step( "Intent Anti-Entropy dispersion" )
+        for i in range(100):
+            correct = True
+            main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
+            for cli in CLIs:
+                onosIds = []
+                ids = cli.getAllIntentsId()
+                onosIds.append( ids )
+                main.log.debug( "Intents in " + cli.name + ": " +
+                                str( sorted( onosIds ) ) )
+                if sorted( ids ) != sorted( intentIds ):
+                    correct = False
+            if correct:
+                break
+            else:
+                time.sleep(1)
+        if not intentStop:
+            intentStop = time.time()
+        global gossipTime
+        gossipTime = intentStop - intentStart
+        main.log.info( "It took about " + str( gossipTime ) +
+                        " seconds for all intents to appear in each node" )
+        # FIXME: make this time configurable/calculate based off of number of
+        #        nodes and gossip rounds
+        utilities.assert_greater_equals(
+                expect=40, actual=gossipTime,
+                onpass="ECM anti-entropy for intents worked within " +
+                       "expected time",
+                onfail="Intent ECM anti-entropy took too long" )
+        if gossipTime <= 40:
+            intentAddResult = True
 
         if not intentAddResult or "key" in pendingMap:
             import time
@@ -561,9 +667,11 @@
         assert numControllers, "numControllers not defined"
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
-        description = " Ping across added host intents"
-        main.log.report( description )
-        main.case( description )
+        main.case( "Verify connectivity by sendind traffic across Intents" )
+        main.caseExplaination = "Ping across added host intents to check " +\
+                                "functionality and check the state of " +\
+                                "the intent"
+        main.step( "Ping across added host intents" )
         PingResult = main.TRUE
         for i in range( 8, 18 ):
             ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -576,7 +684,7 @@
                 main.log.info( "Ping test passed!" )
                 # Don't set PingResult or you'd override failures
         if PingResult == main.FALSE:
-            main.log.report(
+            main.log.error(
                 "Intents have not been installed correctly, pings failed." )
             # TODO: pretty print
             main.log.warn( "ONOS1 intents: " )
@@ -588,92 +696,108 @@
                                            separators=( ',', ': ' ) ) )
             except ( ValueError, TypeError ):
                 main.log.warn( repr( tmpIntents ) )
-        if PingResult == main.TRUE:
-            main.log.report(
-                "Intents have been installed correctly and verified by pings" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=PingResult,
             onpass="Intents have been installed correctly and pings work",
             onfail="Intents have not been installed correctly, pings failed." )
 
+        main.step( "Check Intent state" )
         installedCheck = True
-        if PingResult is not main.TRUE:
-            # Print the intent states
-            intents = main.ONOScli1.intents()
-            intentStates = []
-            main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
-            count = 0
-            # Iter through intents of a node
-            try:
-                for intent in json.loads( intents ):
-                    state = intent.get( 'state', None )
-                    if "INSTALLED" not in state:
-                        installedCheck = False
-                    intentId = intent.get( 'id', None )
-                    intentStates.append( ( intentId, state ) )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing intents." )
-            intentStates.sort()
-            for i, s in intentStates:
-                count += 1
-                main.log.info( "%-6s%-15s%-15s" %
-                               ( str( count ), str( i ), str( s ) ) )
-            leaders = main.ONOScli1.leaders()
-            try:
-                if leaders:
-                    parsedLeaders = json.loads( leaders )
-                    main.log.warn( json.dumps( parsedLeaders,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # check for all intent partitions
-                    # check for election
-                    topics = []
-                    for i in range( 14 ):
-                        topics.append( "intent-partition-" + str( i ) )
-                    # FIXME: this should only be after we start the app
-                    topics.append( "org.onosproject.election" )
-                    main.log.debug( topics )
-                    ONOStopics = [ j['topic'] for j in parsedLeaders ]
-                    for topic in topics:
-                        if topic not in ONOStopics:
-                            main.log.error( "Error: " + topic +
-                                            " not in leaders" )
-                else:
-                    main.log.error( "leaders() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing leaders" )
-                main.log.error( repr( leaders ) )
-            partitions = main.ONOScli1.partitions()
-            try:
-                if partitions :
-                    parsedPartitions = json.loads( partitions )
-                    main.log.warn( json.dumps( parsedPartitions,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # TODO check for a leader in all paritions
-                    # TODO check for consistency among nodes
-                else:
-                    main.log.error( "partitions() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing partitions" )
-                main.log.error( repr( partitions ) )
-            pendingMap = main.ONOScli1.pendingMap()
-            try:
-                if pendingMap :
-                    parsedPending = json.loads( pendingMap )
-                    main.log.warn( json.dumps( parsedPending,
-                                               sort_keys=True,
-                                               indent=4,
-                                               separators=( ',', ': ' ) ) )
-                    # TODO check something here?
-                else:
-                    main.log.error( "pendingMap() returned None" )
-            except ( ValueError, TypeError ):
-                main.log.exception( "Error parsing pending map" )
-                main.log.error( repr( pendingMap ) )
+        # Print the intent states
+        intents = main.ONOScli1.intents()
+        intentStates = []
+        main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+        count = 0
+        # Iter through intents of a node
+        try:
+            for intent in json.loads( intents ):
+                state = intent.get( 'state', None )
+                if "INSTALLED" not in state:
+                    installedCheck = False
+                intentId = intent.get( 'id', None )
+                intentStates.append( ( intentId, state ) )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing intents." )
+        # Print states
+        intentStates.sort()
+        for i, s in intentStates:
+            count += 1
+            main.log.info( "%-6s%-15s%-15s" %
+                           ( str( count ), str( i ), str( s ) ) )
+        utilities.assert_equals( expect=True, actual=installedCheck,
+                                 onpass="Intents are all INSTALLED",
+                                 onfail="Intents are not all in " +
+                                        "INSTALLED state" )
+
+        main.step( "Check leadership of topics" )
+        leaders = main.ONOScli1.leaders()
+        topicCheck = main.TRUE
+        try:
+            if leaders:
+                parsedLeaders = json.loads( leaders )
+                main.log.warn( json.dumps( parsedLeaders,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # check for all intent partitions
+                # check for election
+                # TODO: Look at Devices as topics now that it uses this system
+                topics = []
+                for i in range( 14 ):
+                    topics.append( "intent-partition-" + str( i ) )
+                # FIXME: this should only be after we start the app
+                # FIXME: topics.append( "org.onosproject.election" )
+                # Print leaders output
+                main.log.debug( topics )
+                ONOStopics = [ j['topic'] for j in parsedLeaders ]
+                for topic in topics:
+                    if topic not in ONOStopics:
+                        main.log.error( "Error: " + topic +
+                                        " not in leaders" )
+                        topicCheck = main.FALSE
+            else:
+                main.log.error( "leaders() returned None" )
+                topicCheck = main.FALSE
+        except ( ValueError, TypeError ):
+            topicCheck = main.FALSE
+            main.log.exception( "Error parsing leaders" )
+            main.log.error( repr( leaders ) )
+            # TODO: Check for a leader of these topics
+        utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+                                 onpass="intent Partitions is in leaders",
+                                 onfail="Some topics were lost " )
+        # Print partitions
+        partitions = main.ONOScli1.partitions()
+        try:
+            if partitions :
+                parsedPartitions = json.loads( partitions )
+                main.log.warn( json.dumps( parsedPartitions,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # TODO check for a leader in all paritions
+                # TODO check for consistency among nodes
+            else:
+                main.log.error( "partitions() returned None" )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing partitions" )
+            main.log.error( repr( partitions ) )
+        # Print Pending Map
+        pendingMap = main.ONOScli1.pendingMap()
+        try:
+            if pendingMap :
+                parsedPending = json.loads( pendingMap )
+                main.log.warn( json.dumps( parsedPending,
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+                # TODO check something here?
+            else:
+                main.log.error( "pendingMap() returned None" )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing pending map" )
+            main.log.error( repr( pendingMap ) )
 
         if not installedCheck:
             main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -754,6 +878,39 @@
             except ( ValueError, TypeError ):
                 main.log.exception( "Error parsing pending map" )
                 main.log.error( repr( pendingMap ) )
+        # Print flowrules
+        main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+        main.step( "Wait a minute then ping again" )
+        # the wait is above
+        PingResult = main.TRUE
+        for i in range( 8, 18 ):
+            ping = main.Mininet1.pingHost( src="h" + str( i ),
+                                           target="h" + str( i + 10 ) )
+            PingResult = PingResult and ping
+            if ping == main.FALSE:
+                main.log.warn( "Ping failed between h" + str( i ) +
+                               " and h" + str( i + 10 ) )
+            elif ping == main.TRUE:
+                main.log.info( "Ping test passed!" )
+                # Don't set PingResult or you'd override failures
+        if PingResult == main.FALSE:
+            main.log.error(
+                "Intents have not been installed correctly, pings failed." )
+            # TODO: pretty print
+            main.log.warn( "ONOS1 intents: " )
+            try:
+                tmpIntents = main.ONOScli1.intents()
+                main.log.warn( json.dumps( json.loads( tmpIntents ),
+                                           sort_keys=True,
+                                           indent=4,
+                                           separators=( ',', ': ' ) ) )
+            except ( ValueError, TypeError ):
+                main.log.warn( repr( tmpIntents ) )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=PingResult,
+            onpass="Intents have been installed correctly and pings work",
+            onfail="Intents have not been installed correctly, pings failed." )
 
     def CASE5( self, main ):
         """
@@ -766,7 +923,6 @@
         # assumes that sts is already in you PYTHONPATH
         from sts.topology.teston_topology import TestONTopology
 
-        main.log.report( "Setting up and gathering data for current state" )
         main.case( "Setting up and gathering data for current state" )
         # The general idea for this test case is to pull the state of
         # ( intents,flows, topology,... ) from each ONOS node
@@ -784,10 +940,11 @@
             onpass="Each device has a master",
             onfail="Some devices don't have a master assigned" )
 
+        main.step( "Get the Mastership of each switch" )
         ONOS1Mastership = main.ONOScli1.roles()
         # TODO: Make this a meaningful check
         if "Error" in ONOS1Mastership or not ONOS1Mastership:
-            main.log.report( "Error in getting ONOS roles" )
+            main.log.error( "Error in getting ONOS roles" )
             main.log.warn(
                 "ONOS1 mastership response: " +
                 repr( ONOS1Mastership ) )
@@ -802,7 +959,7 @@
         ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
         intentCheck = main.FALSE
         if "Error" in ONOS1Intents or not ONOS1Intents:
-            main.log.report( "Error in getting ONOS intents" )
+            main.log.error( "Error in getting ONOS intents" )
             main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
         else:
             intentCheck = main.TRUE
@@ -813,7 +970,7 @@
         flowCheck = main.FALSE
         ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
         if "Error" in ONOS1Flows or not ONOS1Flows:
-            main.log.report( "Error in getting ONOS flows" )
+            main.log.error( "Error in getting ONOS flows" )
             main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
         else:
             # TODO: Do a better check, maybe compare flows on switches?
@@ -832,17 +989,9 @@
 
         main.step( "Create TestONTopology object" )
         ctrls = []
-        count = 1
-        temp = ()
-        temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
-        temp = temp + ( "ONOS" + str( count ), )
-        temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
-        temp = temp + \
-            ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
+        temp = ( nodes[0], nodes[0].name, nodes[0].ip_address, 6633 )
         ctrls.append( temp )
-        MNTopo = TestONTopology(
-            main.Mininet1,
-            ctrls )  # can also add Intent API info for intent operations
+        MNTopo = TestONTopology( main.Mininet1, ctrls )
 
         main.step( "Collecting topology information from ONOS" )
         devices = []
@@ -855,18 +1004,30 @@
         links.append( main.ONOScli1.links() )
         clusters = []
         clusters.append( main.ONOScli1.clusters() )
+
+        main.step( "Each host has an IP address" )
         ipResult = main.TRUE
         for controller in range( 0, len( hosts ) ):
             controllerStr = str( controller + 1 )
             for host in hosts[ controller ]:
-                if host is None or host.get( 'ips', [] ) == []:
+                if host is None or host.get( 'ipAddresses', [] ) == []:
                     main.log.error(
                         "DEBUG:Error with host ips on controller" +
                         controllerStr + ": " + str( host ) )
                     ipResult = main.FALSE
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=ipResult,
+            onpass="The ips of the hosts aren't empty",
+            onfail="The ip of at least one host is missing" )
 
         # there should always only be one cluster
-        numClusters = len( json.loads( clusters[ 0 ] ) )
+        main.step( "There is only one dataplane cluster" )
+        try:
+            numClusters = len( json.loads( clusters[ 0 ] ) )
+        except ( ValueError, TypeError ):
+            main.log.exception( "Error parsing clusters[0]: " +
+                                repr( clusters[ 0 ] ) )
         clusterResults = main.FALSE
         if numClusters == 1:
             clusterResults = main.TRUE
@@ -945,13 +1106,6 @@
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
 
-        finalAssert = main.TRUE
-        finalAssert = finalAssert and topoResult and flowCheck \
-                      and intentCheck and consistentMastership and rolesNotNull
-        utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
-                                 onpass="State check successful",
-                                 onfail="State check NOT successful" )
-
     def CASE6( self, main ):
         """
         The Failure case.
@@ -961,10 +1115,22 @@
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
 
-        main.log.report( "Restart ONOS node" )
-        main.log.case( "Restart ONOS node" )
-        main.ONOSbench.onosKill( ONOS1Ip )
+        # Reset non-persistent variables
+        try:
+            iCounterValue = 0
+        except NameError:
+            main.log.error( "iCounterValue not defined, setting to 0" )
+            iCounterValue = 0
+
+        main.case( "Restart ONOS node" )
+        main.caseExplaination = "Killing ONOS process and restart cli " +\
+                                "sessions once onos is up."
+        main.step( "Killing ONOS processes" )
+        killResult = main.ONOSbench.onosKill( ONOS1Ip )
         start = time.time()
+        utilities.assert_equals( expect=main.TRUE, actual=killResult,
+                                 onpass="ONOS Killed",
+                                 onfail="Error killing ONOS" )
 
         main.step( "Checking if ONOS is up yet" )
         count = 0
@@ -975,17 +1141,25 @@
                 break
             else:
                 count = count + 1
+        utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
+                                 onpass="ONOS is back up",
+                                 onfail="ONOS failed to start" )
 
-        cliResult = main.ONOScli1.startOnosCli( ONOS1Ip )
+        main.log.step( "Starting ONOS CLI sessions" )
+        cliResults = main.ONOScli1.startOnosCli( ONOS1Ip )
+        utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+                                 onpass="ONOS cli startup successful",
+                                 onfail="ONOS cli startup failed" )
 
-        caseResults = main.TRUE and onos1Isup and cliResult
-        utilities.assert_equals( expect=main.TRUE, actual=caseResults,
-                                 onpass="ONOS restart successful",
-                                 onfail="ONOS restart NOT successful" )
         if elapsed:
             main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
                            str( elapsed ) )
+            main.restartTime = elapsed
+        else:
+            main.restartTime = -1
         time.sleep( 5 )
+        # rerun on election apps
+        main.ONOScli1.electionTestRun()
 
     def CASE7( self, main ):
         """
@@ -996,7 +1170,7 @@
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         main.case( "Running ONOS Constant State Tests" )
-
+        main.step( "Check that each switch has a master" )
         # Assert that each device has a master
         rolesNotNull = main.ONOScli1.rolesNotNull()
         utilities.assert_equals(
@@ -1009,14 +1183,12 @@
         ONOS1Mastership = main.ONOScli1.roles()
         # FIXME: Refactor this whole case for single instance
         if "Error" in ONOS1Mastership or not ONOS1Mastership:
-            main.log.report( "Error in getting ONOS mastership" )
+            main.log.error( "Error in getting ONOS mastership" )
             main.log.warn( "ONOS1 mastership response: " +
                            repr( ONOS1Mastership ) )
             consistentMastership = main.FALSE
         else:
             consistentMastership = main.TRUE
-            main.log.report(
-                "Switch roles are consistent across all ONOS nodes" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=consistentMastership,
@@ -1042,8 +1214,6 @@
             else:
                 main.log.warn( "Mastership of switch %s changed" % switchDPID )
                 mastershipCheck = main.FALSE
-        if mastershipCheck == main.TRUE:
-            main.log.report( "Mastership of Switches was not changed" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=mastershipCheck,
@@ -1055,11 +1225,11 @@
         ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
         intentCheck = main.FALSE
         if "Error" in ONOS1Intents or not ONOS1Intents:
-            main.log.report( "Error in getting ONOS intents" )
+            main.log.error( "Error in getting ONOS intents" )
             main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
         else:
             intentCheck = main.TRUE
-            main.log.report( "Intents are consistent across all ONOS nodes" )
+            main.log.error( "Intents are consistent across all ONOS nodes" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=intentCheck,
@@ -1084,21 +1254,41 @@
         main.step( "Compare current intents with intents before the failure" )
         # NOTE: this requires case 5 to pass for intentState to be set.
         #      maybe we should stop the test if that fails?
-        sameIntents = main.TRUE
-        if intentState and intentState == ONOS1Intents:
+        sameIntents = main.FALSE
+        if intentState and intentState == ONOSIntents[ 0 ]:
             sameIntents = main.TRUE
-            main.log.report( "Intents are consistent with before failure" )
+            main.log.info( "Intents are consistent with before failure" )
         # TODO: possibly the states have changed? we may need to figure out
-        # what the aceptable states are
-        else:
+        #       what the acceptable states are
+        elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+            sameIntents = main.TRUE
             try:
-                main.log.warn( "ONOS1 intents: " )
-                print json.dumps( json.loads( ONOS1Intents ),
-                                  sort_keys=True, indent=4,
-                                  separators=( ',', ': ' ) )
-            except Exception:
-                pass
-            sameIntents = main.FALSE
+                before = json.loads( intentState )
+                after = json.loads( ONOSIntents[ 0 ] )
+                for intent in before:
+                    if intent not in after:
+                        sameIntents = main.FALSE
+                        main.log.debug( "Intent is not currently in ONOS " +\
+                                        "(at least in the same form):" )
+                        main.log.debug( json.dumps( intent ) )
+            except ( ValueError, TypeError ):
+                main.log.exception( "Exception printing intents" )
+                main.log.debug( repr( ONOSIntents[0] ) )
+                main.log.debug( repr( intentState ) )
+        if sameIntents == main.FALSE:
+            try:
+                main.log.debug( "ONOS intents before: " )
+                main.log.debug( json.dumps( json.loads( intentState ),
+                                            sort_keys=True, indent=4,
+                                            separators=( ',', ': ' ) ) )
+                main.log.debug( "Current ONOS intents: " )
+                main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+                                            sort_keys=True, indent=4,
+                                            separators=( ',', ': ' ) ) )
+            except ( ValueError, TypeError ):
+                main.log.exception( "Exception printing intents" )
+                main.log.debug( repr( ONOSIntents[0] ) )
+                main.log.debug( repr( intentState ) )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=sameIntents,
@@ -1121,14 +1311,13 @@
             if FlowTables == main.FALSE:
                 main.log.info( "Differences in flow table for switch: s" +
                                str( i + 1 ) )
-        if FlowTables == main.TRUE:
-            main.log.report( "No changes were found in the flow tables" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=FlowTables,
             onpass="No changes were found in the flow tables",
             onfail="Changes were found in the flow tables" )
 
+        main.step( "Leadership Election is still functional" )
         # Test of LeadershipElection
 
         leader = ONOS1Ip
@@ -1144,36 +1333,23 @@
                 # all is well
                 pass
             elif leaderN == main.FALSE:
-                # error in  response
-                main.log.report( "Something is wrong with " +
+                # error in response
+                main.log.error( "Something is wrong with " +
                                  "electionTestLeader function, check the" +
                                  " error logs" )
                 leaderResult = main.FALSE
             elif leader != leaderN:
                 leaderResult = main.FALSE
-                main.log.report( "ONOS" + str( controller ) + " sees " +
+                main.log.error( "ONOS" + str( controller ) + " sees " +
                                  str( leaderN ) +
                                  " as the leader of the election app. " +
                                  "Leader should be " + str( leader ) )
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a new " +
-                             "leader was re-elected if applicable )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
             onpass="Leadership election passed",
             onfail="Something went wrong with Leadership election" )
 
-        result = ( mastershipCheck and intentCheck and FlowTables and
-                   rolesNotNull and leaderResult )
-        result = int( result )
-        if result == main.TRUE:
-            main.log.report( "Constant State Tests Passed" )
-        utilities.assert_equals( expect=main.TRUE, actual=result,
-                                 onpass="Constant State Tests Passed",
-                                 onfail="Constant state tests failed" )
-
     def CASE8( self, main ):
         """
         Compare topo
@@ -1189,15 +1365,24 @@
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
 
-        description = "Compare ONOS Topology view to Mininet topology"
-        main.case( description )
-        main.log.report( description )
+        main.case( "Compare ONOS Topology view to Mininet topology" )
+        main.caseExplaination = "Compare topology objects between Mininet" +\
+                                " and ONOS"
         main.step( "Create TestONTopology object" )
-        ctrls = []
-        node = main.ONOS1
-        temp = ( node, node.name, node.ip_address, 6633 )
-        ctrls.append( temp )
-        MNTopo = TestONTopology( main.Mininet1, ctrls )
+        try:
+            ctrls = []
+            node = main.ONOS1
+            temp = ( node, node.name, node.ip_address, 6633 )
+            ctrls.append( temp )
+            MNTopo = TestONTopology( main.Mininet1, ctrls )
+        except Exception:
+            objResult = main.FALSE
+        else:
+            objResult = main.TRUE
+        utilities.assert_equals( expect=main.TRUE, actual=objResult,
+                                 onpass="Created TestONTopology object",
+                                 onfail="Exception while creating " +
+                                        "TestONTopology object" )
 
         main.step( "Comparing ONOS topology to MN" )
         devicesResults = main.TRUE
@@ -1224,7 +1409,7 @@
             for controller in range( 0, len( hosts ) ):
                 controllerStr = str( controller + 1 )
                 for host in hosts[ controller ]:
-                    if host is None or host.get( 'ips', [] ) == []:
+                    if host is None or host.get( 'ipAddresses', [] ) == []:
                         main.log.error(
                             "DEBUG:Error with host ips on controller" +
                             controllerStr + ": " + str( host ) )
@@ -1325,8 +1510,6 @@
         utilities.assert_equals( expect=main.TRUE, actual=topoResult,
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
-        if topoResult == main.TRUE:
-            main.log.report( "ONOS topology view matches Mininet topology" )
 
     def CASE9( self, main ):
         """
@@ -1342,7 +1525,6 @@
 
         description = "Turn off a link to ensure that Link Discovery " +\
                       "is working properly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Kill Link between s3 and s28" )
@@ -1369,7 +1551,6 @@
 
         description = "Restore a link to ensure that Link Discovery is " + \
                       "working properly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Bring link between s3 and s28 back up" )
@@ -1395,14 +1576,13 @@
         switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
 
         description = "Killing a switch to ensure it is discovered correctly"
-        main.log.report( description )
         main.case( description )
         switch = main.params[ 'kill' ][ 'switch' ]
         switchDPID = main.params[ 'kill' ][ 'dpid' ]
 
         # TODO: Make this switch parameterizable
         main.step( "Kill " + switch )
-        main.log.report( "Deleting " + switch )
+        main.log.info( "Deleting " + switch )
         main.Mininet1.delSwitch( switch )
         main.log.info( "Waiting " + str( switchSleep ) +
                        " seconds for switch down to be discovered" )
@@ -1439,11 +1619,9 @@
         switchDPID = main.params[ 'kill' ][ 'dpid' ]
         links = main.params[ 'kill' ][ 'links' ].split()
         description = "Adding a switch to ensure it is discovered correctly"
-        main.log.report( description )
         main.case( description )
 
         main.step( "Add back " + switch )
-        main.log.report( "Adding back " + switch )
         main.Mininet1.addSwitch( switch, dpid=switchDPID )
         for peer in links:
             main.Mininet1.addLink( switch, peer )
@@ -1476,9 +1654,7 @@
         colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
                    'blue': '\033[94m', 'green': '\033[92m',
                    'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
-        description = "Test Cleanup"
-        main.log.report( description )
-        main.case( description )
+        main.case( "Test Cleanup" )
         main.step( "Killing tcpdumps" )
         main.Mininet2.stopTcpdump()
 
@@ -1521,20 +1697,28 @@
         time.sleep( 10 )
 
         main.step( "Stopping Mininet" )
-        main.Mininet1.stopNet()
+        mnResult = main.Mininet1.stopNet()
+        utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+                                 onpass="Mininet stopped",
+                                 onfail="MN cleanup NOT successful" )
 
         main.step( "Checking ONOS Logs for errors" )
         print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
             colors[ 'end' ]
-        print main.ONOSbench.checkLogs( ONOS1Ip )
+        print main.ONOSbench.checkLogs( ONOS1Ip, restart=True )
 
         main.step( "Packing and rotating pcap archives" )
         os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
 
-        # TODO: actually check something here
-        utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
-                                 onpass="Test cleanup successful",
-                                 onfail="Test cleanup NOT successful" )
+        try:
+            timerLog = open( main.logdir + "/Timers.csv", 'w')
+            # Overwrite with empty line and close
+            labels = "Gossip Intents, Restart"
+            data = str( gossipTime ) + ", " + str( main.restartTime )
+            timerLog.write( labels + "\n" + data )
+            timerLog.close()
+        except NameError, e:
+            main.log.exception(e)
 
     def CASE14( self, main ):
         """
@@ -1544,9 +1728,17 @@
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
 
-        leaderResult = main.TRUE
-        main.log.info( "Install leadership election app" )
-        main.ONOScli1.activateApp( "org.onosproject.election" )
+        main.case("Start Leadership Election app")
+        main.step( "Install leadership election app" )
+        appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=appResult,
+            onpass="Election app installed",
+            onfail="Something went wrong with installing Leadership election" )
+
+        main.step( "Run for election on each node" )
+        leaderResult = main.ONOScli1.electionTestRun()
         # check for leader
         leader = main.ONOScli1.electionTestLeader()
         # verify leader is ONOS1
@@ -1555,32 +1747,27 @@
             pass
         elif leader is None:
             # No leader elected
-            main.log.report( "No leader was elected" )
+            main.log.error( "No leader was elected" )
             leaderResult = main.FALSE
         elif leader == main.FALSE:
             # error in  response
             # TODO: add check for "Command not found:" in the driver, this
             # means the app isn't loaded
-            main.log.report( "Something is wrong with electionTestLeader" +
+            main.log.error( "Something is wrong with electionTestLeader" +
                              " function, check the error logs" )
             leaderResult = main.FALSE
         else:
             # error in  response
-            main.log.report(
+            main.log.error(
                 "Unexpected response from electionTestLeader function:'" +
                 str( leader ) +
                 "'" )
             leaderResult = main.FALSE
-
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( consistent " +
-                             "view of leader across listeners and a leader " +
-                             "was elected )" )
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
-            onpass="Leadership election passed",
-            onfail="Something went wrong with Leadership election" )
+            onpass="Successfully ran for leadership",
+            onfail="Failed to run for leadership" )
 
     def CASE15( self, main ):
         """
@@ -1591,7 +1778,6 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         leaderResult = main.TRUE
         description = "Check that Leadership Election is still functional"
-        main.log.report( description )
         main.case( description )
         main.step( "Find current leader and withdraw" )
         leader = main.ONOScli1.electionTestLeader()
@@ -1600,40 +1786,40 @@
         if leader == ONOS1Ip:
             oldLeader = getattr( main, "ONOScli1" )
         elif leader is None or leader == main.FALSE:
-            main.log.report(
+            main.log.error(
                 "Leader for the election app should be an ONOS node," +
                 "instead got '" + str( leader ) + "'" )
             leaderResult = main.FALSE
             oldLeader = None
         else:
             main.log.error( "Leader election --- why am I HERE?!?")
+            leaderResult = main.FALSE
+            oldLeader = None
         if oldLeader:
             withdrawResult = oldLeader.electionTestWithdraw()
         utilities.assert_equals(
             expect=main.TRUE,
             actual=withdrawResult,
-            onpass="App was withdrawn from election",
-            onfail="App was not withdrawn from election" )
+            onpass="Node was withdrawn from election",
+            onfail="Node was not withdrawn from election" )
 
         main.step( "Make sure new leader is elected" )
         leaderN = main.ONOScli1.electionTestLeader()
         if leaderN == leader:
-            main.log.report( "ONOS still sees " + str( leaderN ) +
+            main.log.error( "ONOS still sees " + str( leaderN ) +
                              " as leader after they withdrew" )
             leaderResult = main.FALSE
         elif leaderN == main.FALSE:
             # error in  response
             # TODO: add check for "Command not found:" in the driver, this
             # means the app isn't loaded
-            main.log.report( "Something is wrong with electionTestLeader " +
+            main.log.error( "Something is wrong with electionTestLeader " +
                              "function, check the error logs" )
             leaderResult = main.FALSE
         elif leaderN is None:
             main.log.info(
                 "There is no leader after the app withdrew from election" )
-        if leaderResult:
-            main.log.report( "Leadership election tests passed( There is no " +
-                             "leader after the old leader resigned )" )
+            leaderResult = main.TRUE
         utilities.assert_equals(
             expect=main.TRUE,
             actual=leaderResult,
@@ -1651,17 +1837,1109 @@
             actual=runResult,
             onpass="App re-ran for election",
             onfail="App failed to run for election" )
-        leader = main.ONOScli1.electionTestLeader()
+
+        main.step( "Node became leader when it ran for election" )
+        afterRun = main.ONOScli1.electionTestLeader()
         # verify leader is ONOS1
-        if leader == ONOS1Ip:
-            leaderResult = main.TRUE
+        if afterRun == ONOS1Ip:
+            afterResult = main.TRUE
         else:
-            leaderResult = main.FALSE
-        # TODO: assert on  run and withdraw results?
+            afterResult = main.FALSE
 
         utilities.assert_equals(
             expect=main.TRUE,
-            actual=leaderResult,
-            onpass="Leadership election passed",
-            onfail="ONOS1's election app was not leader after it re-ran " +
-                   "for election" )
+            actual=afterResult,
+            onpass="Old leader successfully re-ran for election",
+            onfail="Something went wrong with Leadership election after " +
+                   "the old leader re-ran for election" )
+
+    def CASE16( self, main ):
+        """
+        Install Distributed Primitives app
+        """
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+
+        # Variables for the distributed primitives tests
+        global pCounterName
+        global iCounterName
+        global pCounterValue
+        global iCounterValue
+        global onosSet
+        global onosSetName
+        pCounterName = "TestON-Partitions"
+        iCounterName = "TestON-inMemory"
+        pCounterValue = 0
+        iCounterValue = 0
+        onosSet = set([])
+        onosSetName = "TestON-set"
+
+        description = "Install Primitives app"
+        main.case( description )
+        main.step( "Install Primitives app" )
+        appName = "org.onosproject.distributedprimitives"
+        appResults = CLIs[0].activateApp( appName )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=appResults,
+                                 onpass="Primitives app activated",
+                                 onfail="Primitives app not activated" )
+
+    def CASE17( self, main ):
+        """
+        Check for basic functionality with distributed primitives
+        """
+        import json
+        # Make sure variables are defined/set
+        assert numControllers, "numControllers not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert CLIs, "CLIs not defined"
+        assert nodes, "nodes not defined"
+        assert pCounterName, "pCounterName not defined"
+        assert iCounterName, "iCounterName not defined"
+        assert onosSetName, "onosSetName not defined"
+        # NOTE: assert fails if value is 0/None/Empty/False
+        try:
+            pCounterValue
+        except NameError:
+            main.log.error( "pCounterValue not defined, setting to 0" )
+            pCounterValue = 0
+        try:
+            iCounterValue
+        except NameError:
+            main.log.error( "iCounterValue not defined, setting to 0" )
+            iCounterValue = 0
+        try:
+            onosSet
+        except NameError:
+            main.log.error( "onosSet not defined, setting to empty Set" )
+            onosSet = set([])
+        # Variables for the distributed primitives tests. These are local only
+        addValue = "a"
+        addAllValue = "a b c d e f"
+        retainValue = "c d e f"
+
+        description = "Check for basic functionality with distributed " +\
+                      "primitives"
+        main.case( description )
+        main.caseExplaination = "Test the methods of the distributed primitives (counters and sets) throught the cli"
+        # DISTRIBUTED ATOMIC COUNTERS
+        main.step( "Increment and get a default counter on each node" )
+        pCounters = []
+        threads = []
+        addedPValues = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counterTestIncrement,
+                             name="counterIncrement-" + str( i ),
+                             args=[ pCounterName ] )
+            pCounterValue += 1
+            addedPValues.append( pCounterValue )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            pCounters.append( t.result )
+        # Check that counter incremented numController times
+        pCounterResults = True
+        for i in addedPValues:
+            tmpResult = i in pCounters
+            pCounterResults = pCounterResults and tmpResult
+            if not tmpResult:
+                main.log.error( str( i ) + " is not in partitioned "
+                                "counter incremented results" )
+        utilities.assert_equals( expect=True,
+                                 actual=pCounterResults,
+                                 onpass="Default counter incremented",
+                                 onfail="Error incrementing default" +
+                                        " counter" )
+
+        main.step( "Increment and get an in memory counter on each node" )
+        iCounters = []
+        addedIValues = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counterTestIncrement,
+                             name="icounterIncrement-" + str( i ),
+                             args=[ iCounterName ],
+                             kwargs={ "inMemory": True } )
+            iCounterValue += 1
+            addedIValues.append( iCounterValue )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            iCounters.append( t.result )
+        # Check that counter incremented numController times
+        iCounterResults = True
+        for i in addedIValues:
+            tmpResult = i in iCounters
+            iCounterResults = iCounterResults and tmpResult
+            if not tmpResult:
+                main.log.error( str( i ) + " is not in the in-memory "
+                                "counter incremented results" )
+        utilities.assert_equals( expect=True,
+                                 actual=iCounterResults,
+                                 onpass="In memory counter incremented",
+                                 onfail="Error incrementing in memory" +
+                                        " counter" )
+
+        main.step( "Check counters are consistant across nodes" )
+        onosCounters = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].counters,
+                             name="counters-" + str( i ) )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            onosCounters.append( t.result )
+        tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+        if all( tmp ):
+            main.log.info( "Counters are consistent across all nodes" )
+            consistentCounterResults = main.TRUE
+        else:
+            main.log.error( "Counters are not consistent across all nodes" )
+            consistentCounterResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=consistentCounterResults,
+                                 onpass="ONOS counters are consistent " +
+                                        "across nodes",
+                                 onfail="ONOS Counters are inconsistent " +
+                                        "across nodes" )
+
+        main.step( "Counters we added have the correct values" )
+        correctResults = main.TRUE
+        for i in range( numControllers ):
+            current = json.loads( onosCounters[i] )
+            pValue = None
+            iValue = None
+            try:
+                for database in current:
+                    partitioned = database.get( 'partitionedDatabaseCounters' )
+                    if partitioned:
+                        for value in partitioned:
+                            if value.get( 'name' ) == pCounterName:
+                                pValue = value.get( 'value' )
+                                break
+                    inMemory = database.get( 'inMemoryDatabaseCounters' )
+                    if inMemory:
+                        for value in inMemory:
+                            if value.get( 'name' ) == iCounterName:
+                                iValue = value.get( 'value' )
+                                break
+            except AttributeError, e:
+                main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
+                                "is not as expected" )
+                correctResults = main.FALSE
+            if pValue == pCounterValue:
+                main.log.info( "Partitioned counter value is correct" )
+            else:
+                main.log.error( "Partitioned counter value is incorrect," +
+                                " expected value: " + str( pCounterValue )
+                                + " current value: " + str( pValue ) )
+                correctResults = main.FALSE
+            if iValue == iCounterValue:
+                main.log.info( "In memory counter value is correct" )
+            else:
+                main.log.error( "In memory counter value is incorrect, " +
+                                "expected value: " + str( iCounterValue ) +
+                                " current value: " + str( iValue ) )
+                correctResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=correctResults,
+                                 onpass="Added counters are correct",
+                                 onfail="Added counters are incorrect" )
+        # DISTRIBUTED SETS
+        main.step( "Distributed Set get" )
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=getResults,
+                                 onpass="Set elements are correct",
+                                 onfail="Set elements are incorrect" )
+
+        main.step( "Distributed Set size" )
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=sizeResults,
+                                 onpass="Set sizes are correct",
+                                 onfail="Set sizes are incorrect" )
+
+        main.step( "Distributed Set add()" )
+        onosSet.add( addValue )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAdd-" + str( i ),
+                             args=[ onosSetName, addValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addResults = main.FALSE
+            else:
+                # unexpected result
+                addResults = main.FALSE
+        if addResults != main.TRUE:
+            main.log.error( "Error executing set add" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addResults = addResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addResults,
+                                 onpass="Set add correct",
+                                 onfail="Set add was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set contains()" )
+        containsResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setContains-" + str( i ),
+                             args=[ onosSetName ],
+                             kwargs={ "values": addValue } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            # NOTE: This is the tuple
+            containsResponses.append( t.result )
+
+        containsResults = main.TRUE
+        for i in range( numControllers ):
+            if containsResponses[ i ] == main.ERROR:
+                containsResults = main.FALSE
+            else:
+                containsResults = containsResults and\
+                                  containsResponses[ i ][ 1 ]
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=containsResults,
+                                 onpass="Set contains is functional",
+                                 onfail="Set contains failed" )
+
+        main.step( "Distributed Set containsAll()" )
+        containsAllResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setContainsAll-" + str( i ),
+                             args=[ onosSetName ],
+                             kwargs={ "values": addAllValue } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            # NOTE: This is the tuple
+            containsAllResponses.append( t.result )
+
+        containsAllResults = main.TRUE
+        for i in range( numControllers ):
+            if containsResponses[ i ] == main.ERROR:
+                containsResults = main.FALSE
+            else:
+                containsResults = containsResults and\
+                                  containsResponses[ i ][ 1 ]
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=containsAllResults,
+                                 onpass="Set containsAll is functional",
+                                 onfail="Set containsAll failed" )
+
+        main.step( "Distributed Set remove()" )
+        onosSet.remove( addValue )
+        removeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestRemove-" + str( i ),
+                             args=[ onosSetName, addValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            removeResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        removeResults = main.TRUE
+        for i in range( numControllers ):
+            if removeResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif removeResponses[ i ] == main.FALSE:
+                # not in set, probably fine
+                pass
+            elif removeResponses[ i ] == main.ERROR:
+                # Error in execution
+                removeResults = main.FALSE
+            else:
+                # unexpected result
+                removeResults = main.FALSE
+        if removeResults != main.TRUE:
+            main.log.error( "Error executing set remove" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        removeResults = removeResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=removeResults,
+                                 onpass="Set remove correct",
+                                 onfail="Set remove was incorrect" )
+
+        main.step( "Distributed Set removeAll()" )
+        onosSet.difference_update( addAllValue.split() )
+        removeAllResponses = []
+        threads = []
+        try:
+            for i in range( numControllers ):
+                t = main.Thread( target=CLIs[i].setTestRemove,
+                                 name="setTestRemoveAll-" + str( i ),
+                                 args=[ onosSetName, addAllValue ] )
+                threads.append( t )
+                t.start()
+            for t in threads:
+                t.join()
+                removeAllResponses.append( t.result )
+        except Exception, e:
+            main.log.exception(e)
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        removeAllResults = main.TRUE
+        for i in range( numControllers ):
+            if removeAllResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif removeAllResponses[ i ] == main.FALSE:
+                # not in set, probably fine
+                pass
+            elif removeAllResponses[ i ] == main.ERROR:
+                # Error in execution
+                removeAllResults = main.FALSE
+            else:
+                # unexpected result
+                removeAllResults = main.FALSE
+        if removeAllResults != main.TRUE:
+            main.log.error( "Error executing set removeAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        removeAllResults = removeAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=removeAllResults,
+                                 onpass="Set removeAll correct",
+                                 onfail="Set removeAll was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set clear()" )
+        onosSet.clear()
+        clearResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestClear-" + str( i ),
+                             args=[ onosSetName, " "],  # Values doesn't matter
+                             kwargs={ "clear": True } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            clearResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        clearResults = main.TRUE
+        for i in range( numControllers ):
+            if clearResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif clearResponses[ i ] == main.FALSE:
+                # Nothing set, probably fine
+                pass
+            elif clearResponses[ i ] == main.ERROR:
+                # Error in execution
+                clearResults = main.FALSE
+            else:
+                # unexpected result
+                clearResults = main.FALSE
+        if clearResults != main.TRUE:
+            main.log.error( "Error executing set clear" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        clearResults = clearResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=clearResults,
+                                 onpass="Set clear correct",
+                                 onfail="Set clear was incorrect" )
+
+        main.step( "Distributed Set addAll()" )
+        onosSet.update( addAllValue.split() )
+        addResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestAdd,
+                             name="setTestAddAll-" + str( i ),
+                             args=[ onosSetName, addAllValue ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            addResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        addAllResults = main.TRUE
+        for i in range( numControllers ):
+            if addResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif addResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif addResponses[ i ] == main.ERROR:
+                # Error in execution
+                addAllResults = main.FALSE
+            else:
+                # unexpected result
+                addAllResults = main.FALSE
+        if addAllResults != main.TRUE:
+            main.log.error( "Error executing set addAll" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " + str( size ) +
+                                " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        addAllResults = addAllResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=addAllResults,
+                                 onpass="Set addAll correct",
+                                 onfail="Set addAll was incorrect" )
+
+        main.step( "Distributed Set retain()" )
+        onosSet.intersection_update( retainValue.split() )
+        retainResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestRemove,
+                             name="setTestRetain-" + str( i ),
+                             args=[ onosSetName, retainValue ],
+                             kwargs={ "retain": True } )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            retainResponses.append( t.result )
+
+        # main.TRUE = successfully changed the set
+        # main.FALSE = action resulted in no change in set
+        # main.ERROR - Some error in executing the function
+        retainResults = main.TRUE
+        for i in range( numControllers ):
+            if retainResponses[ i ] == main.TRUE:
+                # All is well
+                pass
+            elif retainResponses[ i ] == main.FALSE:
+                # Already in set, probably fine
+                pass
+            elif retainResponses[ i ] == main.ERROR:
+                # Error in execution
+                retainResults = main.FALSE
+            else:
+                # unexpected result
+                retainResults = main.FALSE
+        if retainResults != main.TRUE:
+            main.log.error( "Error executing set retain" )
+
+        # Check if set is still correct
+        size = len( onosSet )
+        getResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestGet,
+                             name="setTestGet-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            getResponses.append( t.result )
+        getResults = main.TRUE
+        for i in range( numControllers ):
+            if isinstance( getResponses[ i ], list):
+                current = set( getResponses[ i ] )
+                if len( current ) == len( getResponses[ i ] ):
+                    # no repeats
+                    if onosSet != current:
+                        main.log.error( "ONOS" + str( i + 1 ) +
+                                        " has incorrect view" +
+                                        " of set " + onosSetName + ":\n" +
+                                        str( getResponses[ i ] ) )
+                        main.log.debug( "Expected: " + str( onosSet ) )
+                        main.log.debug( "Actual: " + str( current ) )
+                        getResults = main.FALSE
+                else:
+                    # error, set is not a set
+                    main.log.error( "ONOS" + str( i + 1 ) +
+                                    " has repeat elements in" +
+                                    " set " + onosSetName + ":\n" +
+                                    str( getResponses[ i ] ) )
+                    getResults = main.FALSE
+            elif getResponses[ i ] == main.ERROR:
+                getResults = main.FALSE
+        sizeResponses = []
+        threads = []
+        for i in range( numControllers ):
+            t = main.Thread( target=CLIs[i].setTestSize,
+                             name="setTestSize-" + str( i ),
+                             args=[ onosSetName ] )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            sizeResponses.append( t.result )
+        sizeResults = main.TRUE
+        for i in range( numControllers ):
+            if size != sizeResponses[ i ]:
+                sizeResults = main.FALSE
+                main.log.error( "ONOS" + str( i + 1 ) +
+                                " expected a size of " +
+                                str( size ) + " for set " + onosSetName +
+                                " but got " + str( sizeResponses[ i ] ) )
+        retainResults = retainResults and getResults and sizeResults
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=retainResults,
+                                 onpass="Set retain correct",
+                                 onfail="Set retain was incorrect" )
+
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.params b/TestON/tests/IntentEventTP/IntentEventTP.params
index fb2e625..ddda23e 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.params
+++ b/TestON/tests/IntentEventTP/IntentEventTP.params
@@ -1,19 +1,20 @@
 <PARAMS>
 
-    <testcases>1,2,1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2,1,2,1,2,1,2</testcases>
     
+    <isOnBaremetal>True</isOnBaremetal>
     <debugMode></debugMode>  #nothing means false 
 
     <ENV>
     <cellName>intentTP</cellName>
-    <cellApps>null,intentperf</cellApps>
+    <cellApps>drivers,null,intentperf</cellApps>
     </ENV>
 
-    <SCALE>1,3,5,7</SCALE>
+    <SCALE>1,3,3,5,5,7,7</SCALE>
     <availableNodes>7</availableNodes>
 
     <GIT>
-        <autopull>off</autopull>
+        <autopull>on</autopull>
         <checkout>master</checkout>
     </GIT>
 
@@ -32,32 +33,33 @@
         <ip4>10.128.5.54</ip4>
         <port4>6633</port4>
 
-        <ip5>10.128.5.65</ip5>
+        <ip5>10.128.5.55</ip5>
         <port5>6633</port5>
 
-        <ip6>10.128.5.66</ip6>
+        <ip6>10.128.5.56</ip6>
         <port6>6633</port6>
 
-         <ip7>10.128.5.67</ip7>
+         <ip7>10.128.5.57</ip7>
         <port7>6633</port7> 
     </CTRL>
 
-    <MN><ip1>10.128.5.55</ip1></MN>
+    <MN><ip1>10.128.5.50</ip1></MN>
 
     <BENCH>
         <user>admin</user>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.128.5.50</ip1>
     </BENCH>
 
     <TEST> 
         <loadFrom>1,1,1,1,1,1,1</loadFrom>                                     #generate load on server, 1 = generator on 
         <numSwitches>10,10,10,10,10,10,10</numSwitches>
         <skipCleanInstall>yes</skipCleanInstall>
-        <duration>30</duration>
+        <duration>400</duration>
         <log_interval>20</log_interval>
-        <numKeys>5000</numKeys>
+        <numKeys>40000</numKeys>
         <cyclePeriod>1000</cyclePeriod>
         <neighbors>0,a</neighbors>           #a == all nodes (-1)
+        <flowRuleBUEnabled>false</flowRuleBUEnabled>
     </TEST>
 
     <METRICS>
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.py b/TestON/tests/IntentEventTP/IntentEventTP.py
index 0e8af7d..99332f7 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.py
+++ b/TestON/tests/IntentEventTP/IntentEventTP.py
@@ -16,6 +16,8 @@
     def CASE1( self, main ):           
                                         
         import time                     
+        import os.path
+
         global init       
         try: 
             if type(init) is not bool: 
@@ -35,7 +37,13 @@
         skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
         cellName = main.params[ 'ENV' ][ 'cellName' ]        
         numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
+        flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
+        onBaremetal = main.params['isOnBaremetal']
+        homeDir = os.path.expanduser('~')
 
+        main.exceptions = [0]*11
+        main.warnings = [0]*11
+        main.errors = [0]*11
 
         # -- INIT SECTION, ONLY RUNS ONCE -- # 
         if init == False: 
@@ -44,7 +52,7 @@
             global ONOSIp                   #list of ONOS IP addresses
             global scale 
             global commit
-            
+
             clusterCount = 0
             ONOSIp = [ 0 ]
             scale = (main.params[ 'SCALE' ]).split(",")            
@@ -109,7 +117,24 @@
             myDistribution.append(numSwitches[node-1])
 
         #main.ONOSbench.createLinkGraphFile( BENCHIp,cellIp,myDistribution)
-       
+
+        if onBaremetal == "True":
+            filename = "/onos/tools/package/bin/onos-service"
+            serviceConfig = open(homeDir + filename, 'w+')
+            serviceConfig.write("#!/bin/bash\n ")
+            serviceConfig.write("#------------------------------------- \n ")
+            serviceConfig.write("# Starts ONOS Apache Karaf container\n ")
+            serviceConfig.write("#------------------------------------- \n ")
+            serviceConfig.write("#export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}\n ")
+            serviceConfig.write("""export JAVA_OPTS="${JAVA_OPTS:--Xms8G -Xmx8G}" \n """)
+            serviceConfig.write("")
+            serviceConfig.write("ONOS_HOME=/opt/onos \n ")
+            serviceConfig.write("")
+            serviceConfig.write("[ -d $ONOS_HOME ] && cd $ONOS_HOME || ONOS_HOME=$(dirname $0)/..\n")
+            serviceConfig.write("""${ONOS_HOME}/apache-karaf-$KARAF_VERSION/bin/karaf "$@" \n """)
+            serviceConfig.close()
+            main.log.info("Set /onos/tools/package/bin/onos-service with 8G Xms/Xmx Options.")
+      
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()  
 
@@ -133,17 +158,39 @@
 
         time.sleep(20)
 
-        #main.ONOSbench.handle.sendline("""onos $OC1 "cfg setorg.onosproject.provider.nil.NullProviders enabled true" """)
-        #main.ONOSbench.handle.expect(":~")
-        #print main.ONOSbench.handle.before
+
+        while True: 
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount """ + str(clusterCount*10) + """ " """)
+            main.ONOSbench.handle.expect(":~")
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
+            main.ONOSbench.handle.expect(":~")
+            if ("value=" + str(clusterCount*10)) in main.ONOSbench.handle.before:
+                main.log.info("Device count set")
+                main.log.info("before" + main.ONOSbench.handle.before)
+                break
+            time.sleep(10)
+            main.log.info("cfg set failure, retrying")
+            main.log.info("before" + main.ONOSbench.handle.before)
         
-        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount """ + str(clusterCount*10) + """ " """)
+        while True:
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
+            main.ONOSbench.handle.expect(":~")
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
+            main.ONOSbench.handle.expect(":~")
+            if ("value=linear") in main.ONOSbench.handle.before:
+                main.log.info("Device count set")
+                main.log.info("before" + main.ONOSbench.handle.before)
+                break
+            time.sleep(10)
+            main.log.info("cfg set failure, retrying")
+            main.log.info("before" + main.ONOSbench.handle.before)
+
+        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBU + """" """)
         main.ONOSbench.handle.expect(":~")
-        print main.ONOSbench.handle.before
-        time.sleep(10)
-        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
+        main.ONOSbench.handle.sendline("""onos $OC1 "cfg get" """)
         main.ONOSbench.handle.expect(":~")
-        print main.ONOSbench.handle.before
+        main.log.info(main.ONOSbench.handle.before)
+
         time.sleep(10)
         main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
         main.ONOSbench.handle.expect(":~")
@@ -159,6 +206,7 @@
         while True:
             main.ONOSbench.handle.sendline("onos $OC1 summary")
             main.ONOSbench.handle.expect(":~")
+            main.log.info("before" + main.ONOSbench.handle.before)
             clusterCheck = ((main.ONOSbench.handle.before).splitlines())[3]
             print("\nBefore: " + str(clusterCheck))
             if ("SCC(s)=1,") in clusterCheck:
@@ -174,7 +222,7 @@
                     break
             lastOutput = clusterCheck
             time.sleep(5)
-
+        main.ONOSbench.onosErrorLog(ONOSIp[1])
 
     def CASE2( self, main ): 
         import time
@@ -182,6 +230,26 @@
         import string
         import csv
         import numpy
+        import os.path
+
+        global currentNeighbors
+        neighbors = []
+
+        try:
+            currentNeighbors
+        except:
+            currentNeighbors = "0"
+            neighbors = ['0']
+        else:
+            if currentNeighbors == "r":      #reset
+                currentNeighbors = "a"
+                neighbors = ['0']
+            else:
+                currentNeighbors = "r" 
+                neighbors = ['a']
+
+        if clusterCount == 1:
+            currentNeighbors = "r"
 
         main.log.info("Cluster Count = " + str(clusterCount))
 
@@ -193,7 +261,7 @@
         debug = main.params[ 'debugMode' ]
         numKeys = main.params[ 'TEST' ][ 'numKeys' ]
         cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
-        neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",") 
+        #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",") 
         metricList = [intentsRate, intentsWithdrawn, intentsFailed]
 
         for n in range(0, len(neighbors)): 
@@ -265,7 +333,7 @@
                 clusterTotal = str(numpy.sum(groupResult))
                 main.log.report("Results from this round of polling: " + str(groupResult))
                 main.log.report("Cluster Total: " + clusterTotal + "\n")
-            
+
             cmd = "onos $OC1 intent-perf-stop"
             main.ONOSbench.handle.sendline(cmd)
             main.ONOSbench.handle.expect(":~")
@@ -284,6 +352,7 @@
                 resultsDB.write(resultString)
             
             resultsDB.close() 
-
+            
+            main.ONOSbench.onosErrorLog(ONOSIp[1])
                         
 
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.topo b/TestON/tests/IntentEventTP/IntentEventTP.topo
index 30a1467..763a4d6 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.topo
+++ b/TestON/tests/IntentEventTP/IntentEventTP.topo
@@ -3,7 +3,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -12,7 +12,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
         </ONOS2cli>
 
         <ONOS3cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
         </ONOS3cli>
 
         <ONOS4cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
         </ONOS4cli>
 
         <ONOS5cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
         </ONOS5cli>
 
         <ONOS6cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
         </ONOS6cli>
 
         <ONOS7cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -112,7 +112,7 @@
 
     
         <ONOS5>
-            <host>10.128.5.65</host>
+            <host>10.128.5.55</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -121,7 +121,7 @@
         </ONOS5>
 
         <ONOS6>
-            <host>10.128.5.66</host>
+            <host>10.128.5.56</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -130,7 +130,7 @@
         </ONOS6>
 
         <ONOS7>
-            <host>10.128.5.67</host>
+            <host>10.128.5.57</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -141,6 +141,4 @@
     </COMPONENT>
 
 </TOPOLOGY>
-
-
-    
+ 
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
index 3185cc2..2b76662 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
@@ -1,22 +1,22 @@
 <PARAMS>
 
-    <testcases>1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2</testcases>
 
-    <SCALE>3,5,7</SCALE>
+    <SCALE>1,3,5,7</SCALE>
     <availableNodes>7</availableNodes>
  
     <ENV>
         <cellName>IntentInstallWithdrawCell</cellName>
-        <cellApps>null</cellApps>
+        <cellApps>drivers,null</cellApps>
     </ENV>
 
     <TEST>
         <skipCleanInstall>yes</skipCleanInstall>
         <switchCount>7</switchCount>
-        <warmUp>4</warmUp>
-        <sampleSize>10</sampleSize>                     
+        <warmUp>10</warmUp>
+        <sampleSize>20</sampleSize>                     
         <wait></wait>
-        <intents>1000</intents>                       #list format, will be split on ','
+        <intents>1,100,1000</intents>                       #list format, will be split on ','
         <debug>True</debug>                                        #"True" for true
     </TEST>
 
@@ -28,36 +28,36 @@
     <CTRL>
         <USER>admin</USER>
         
-        <ip1>10.128.5.51</ip1>
+        <ip1>10.254.1.201</ip1>
         <port1>6633</port1>
         
-        <ip2>10.128.5.52</ip2>
+        <ip2>10.254.1.202</ip2>
         <port2>6633</port2>
         
-        <ip3>10.128.5.53</ip3>
+        <ip3>10.254.1.203</ip3>
         <port3>6633</port3>
         
-        <ip4>10.128.5.54</ip4>
+        <ip4>10.254.1.204</ip4>
         <port4>6633</port4>
         
-        <ip5>10.128.5.65</ip5>
+        <ip5>10.254.1.205</ip5>
         <port5>6633</port5>
         
-        <ip6>10.128.5.66</ip6>
+        <ip6>10.254.1.206</ip6>
         <port6>6633</port6> 
        
-        <ip7>10.128.5.67</ip7>
+        <ip7>10.254.1.207</ip7>
         <port7>6633</port7>
 
     </CTRL>
 
     <MN>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.254.1.200</ip1>
     </MN>
 
     <BENCH>
         <user>admin</user>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.254.1.200</ip1>
     </BENCH>
 
     <JSON>
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
index be8f38c..83dac2a 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
@@ -252,6 +252,6 @@
             resultString += str(numpy.std(installed)) + ","
             resultString += str(numpy.mean(withdrawn)) + ","
             resultString += str(numpy.std(withdrawn)) + "\n"
-            resultsDB = open("IntentInstallWithdrawLatDB", "w+")
+            resultsDB = open("IntentInstallWithdrawLatDB", "a")
             resultsDB.write(resultString)
             resultsDB.close()
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
index 30a1467..0e45e0f 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
@@ -3,7 +3,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -12,7 +12,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
         </ONOS2cli>
 
         <ONOS3cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
         </ONOS3cli>
 
         <ONOS4cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
         </ONOS4cli>
 
         <ONOS5cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
         </ONOS5cli>
 
         <ONOS6cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
         </ONOS6cli>
 
         <ONOS7cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -75,7 +75,7 @@
         </ONOS7cli>
 
         <ONOS1>
-            <host>10.128.5.51</host>
+            <host>10.254.1.201</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -84,7 +84,7 @@
         </ONOS1>
 
         <ONOS2>
-            <host>10.128.5.52</host>
+            <host>10.254.1.202</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -93,7 +93,7 @@
         </ONOS2>
 
         <ONOS3>
-            <host>10.128.5.53</host>
+            <host>10.254.1.203</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -102,7 +102,7 @@
         </ONOS3>
 
         <ONOS4>
-            <host>10.128.5.54</host>
+            <host>10.254.1.204</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -112,7 +112,7 @@
 
     
         <ONOS5>
-            <host>10.128.5.65</host>
+            <host>10.254.1.205</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -121,7 +121,7 @@
         </ONOS5>
 
         <ONOS6>
-            <host>10.128.5.66</host>
+            <host>10.254.1.206</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -130,7 +130,7 @@
         </ONOS6>
 
         <ONOS7>
-            <host>10.128.5.67</host>
+            <host>10.254.1.207</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.params b/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
index 27af1f2..8e7a6a4 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
@@ -1,27 +1,28 @@
 <PARAMS>
 
-    <testcases>1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2</testcases>
 
-    <SCALE>1</SCALE>
+    <SCALE>1,3,5,7</SCALE>
     <availableNodes>7</availableNodes>
  
     <ENV>
         <cellName>intentRerouteCell</cellName>
-        <cellApps>null,intentperf,metrics</cellApps>
-        <scale1switches>8</scale1switches>
-        <scale3switches>2,2,3</scale3switches>
-        <scale5switches>2,2,1,1,1</scale5switches>
-        <scale7switches>1,1,1,1,1,1,1</scale7switches>
-    
+        <cellApps>drivers,null,intentperf,metrics</cellApps>
     </ENV>
 
     <TEST>
         <skipCleanInstall>yes</skipCleanInstall>
-        <warmUp>1</warmUp>
-        <sampleSize>2</sampleSize>                     
+        <warmUp>5</warmUp>
+        <sampleSize>10</sampleSize>                     
         <wait></wait>
-        <intents>1,10</intents>                       #list format, will be split on ','
-        <debug>True</debug>                          
+        <intents>1,100,1000</intents>                       #list format, will be split on ','
+        <debug>True</debug>     
+       
+        <s1>1,1,1,1,1,1,1,1</s1>
+        <s3>2,2,1,1,3,3,3,1</s3>
+        <s5>2,2,1,1,3,4,5,3</s5>
+        <s7>2,3,1,1,5,6,7,4</s7>
+
     </TEST>
 
     <METRICS>
@@ -40,36 +41,36 @@
     <CTRL>
         <USER>admin</USER>
         
-        <ip1>10.128.5.51</ip1>
+        <ip1>10.254.1.201</ip1>
         <port1>6633</port1>
         
-        <ip2>10.128.5.52</ip2>
+        <ip2>10.254.1.202</ip2>
         <port2>6633</port2>
         
-        <ip3>10.128.5.53</ip3>
+        <ip3>10.254.1.203</ip3>
         <port3>6633</port3>
         
-        <ip4>10.128.5.54</ip4>
+        <ip4>10.254.1.204</ip4>
         <port4>6633</port4>
         
-        <ip5>10.128.5.65</ip5>
+        <ip5>10.254.1.205</ip5>
         <port5>6633</port5>
         
-        <ip6>10.128.5.66</ip6>
+        <ip6>10.254.1.206</ip6>
         <port6>6633</port6> 
        
-        <ip7>10.128.5.67</ip7>
+        <ip7>10.254.1.207</ip7>
         <port7>6633</port7>
 
     </CTRL>
 
     <MN>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.254.1.200</ip1>
     </MN>
 
     <BENCH>
         <user>admin</user>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.254.1.200</ip1>
     </BENCH>
 
     <JSON>
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.py b/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
index 84d5c03..f824a5c 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
@@ -79,9 +79,6 @@
         clusterCount = int(scale[0])
         scale.remove(scale[0])       
        
-        switchParams = ("scale" + str(clusterCount) + "switches")
-        switchCount = (main.params[ 'ENV' ][ switchParams ]).split(",")
-
         #kill off all onos processes 
         main.log.step("Safety check, killing all ONOS processes")
         main.log.step("before initiating enviornment setup")
@@ -105,15 +102,6 @@
         main.step( "Set Cell" )
         main.ONOSbench.setCell(cellName)
         
-        if clusterCount == 1:
-            temp = "one"
-        if clusterCount == 3:
-            temp = "three"
-        if clusterCount == 5:
-            temp = "five"
-        if clusterCount == 7:
-            temp = "seven"
-
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()  
 
@@ -134,7 +122,10 @@
             if not isup:
                 main.log.report( "ONOS " + str(node) + " didn't start!" )
         main.log.info("Startup sequence complete")
-     
+    
+        deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
+        print("Device mastership list: " + str(deviceMastership))
+
         main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount 8 " """)
         main.ONOSbench.handle.expect(":~")
         print repr(main.ONOSbench.handle.before)
@@ -143,53 +134,49 @@
         main.ONOSbench.handle.expect(":~")
         print repr(main.ONOSbench.handle.before)
         time.sleep(3)
-        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled true" """)
-        main.ONOSbench.handle.expect(":~")
-        print repr(main.ONOSbench.handle.before) 
 
-        while True: 
+        time.sleep(10)
+
+        once = True
+        for attempt in range(0,10): 
             main.ONOSbench.handle.sendline("onos $OC1 summary")
             main.ONOSbench.handle.expect(":~")
             x = main.ONOSbench.handle.before
-            if "devices=8" in x:
+            main.log.info("Initial setup. ONOS summary response: \n" + x + "\n")
+
+            if "devices=8" in x and "links=16," in x: 
                 break
-            else:   
+            else:                
+                if once == False:
+                    main.log.error("Null provider start failed, retrying..")
+                once = False
                 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled false" """)
                 main.ONOSbench.handle.expect(":~")
                 time.sleep(3)
                 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled true" """)
                 main.ONOSbench.handle.expect(":~")
-                main.log.error("Null provider start failed, retrying..") 
-                time.sleep(8)
+                time.sleep(8) 
 
-        main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
-        main.ONOSbench.handle.expect(":~")
-        print main.ONOSbench.handle.before
-        time.sleep(10)
-        main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
-        main.ONOSbench.handle.expect(":~")
-        
-        temp = 1 
-        for node in range(1, clusterCount + 1): 
-            for switch in range (0, int(switchCount[node-1])): 
-                cmd = ("""onos $OC1 "device-role null:000000000000000""" + str(temp) + " " + ONOSIp[node] + """ master" """)
+        index = 1
+        for node in deviceMastership:
+            for attempt in range(0,10):
+                cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)]  + """ master" """)
+                main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
                 main.ONOSbench.handle.sendline(cmd)
-                main.log.info( cmd ) 
                 main.ONOSbench.handle.expect(":~")
-                temp += 1
+                time.sleep(4)
+                
+                cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index)) 
+                main.log.info(cmd) 
+                main.ONOSbench.handle.sendline(cmd)
+                main.ONOSbench.handle.expect(":~")
+                check = main.ONOSbench.handle.before
+                main.log.info("CHECK:\n" + check)
+                if ("master=" + ONOSIp[int(node)]) in check:
+                    break
+            index += 1
 
 
-        #    cmd = ( """onos $OC1 "device-role null:0000000000000008 10.128.5.52 master" """)
-        #if clusterCount == 7: 
-        #    cmd = ( """onos $OC1 "device-role null:0000000000000008 10.128.5.53 master" """)
-            
-        #main.ONOSbench.handle.sendline(cmd)
-        #main.log.info( cmd )
-        #main.ONOSbench.handle.expect(":~")
-
-        #print "sleeping"
-        #time.sleep(120)
-
     def CASE2( self, main ):
          
         import time
@@ -226,7 +213,7 @@
             debug = False
 
         ingress = "null:0000000000000001"
-        egress = "null:0000000000000008"
+        egress = "null:0000000000000007"
 
         for intents in intentsList:
             main.log.report("Intent Batch size: " + str(intents) + "\n      ")
@@ -236,7 +223,7 @@
                 if run > warmUp:
                     main.log.info("Starting test iteration " + str(run-warmUp))
 
-                cmd = """onos $OC1 push-test-intents -i" """
+                cmd = """onos $OC1 "push-test-intents -i """
                 cmd += ingress + "/0 "
                 cmd += egress + "/0 "
                 cmd += str(intents) +""" 1" """
@@ -253,10 +240,10 @@
                     main.ONOSbench.handle.sendline("onos $OC1 summary")
                     main.ONOSbench.handle.expect(":~")
                     linkCheck = main.ONOSbench.handle.before
-                    if ("flows=16,") in linkCheck:
+                    if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
                         break
                     if i == 39:
-                        main.log.error("Flow count incorrect, data invalid."+ linkCheck)
+                        main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
 
 
                 #collect timestamp from link cut
@@ -270,7 +257,8 @@
                     time.sleep(2)
                     main.ONOSbench.handle.expect(":~")
                     raw = main.ONOSbench.handle.before
-                    if "NullLinkProvider" in raw and "links=14" in raw:
+                    #if "NullLinkProvider" in raw and "links=14" in raw:
+                    if "links=14" in raw:
                         break
                     if i >= 9:
                         main.log.error("Expected output not being recieved... continuing")
@@ -292,22 +280,27 @@
                     main.ONOSbench.handle.sendline("onos $OC1 summary")
                     main.ONOSbench.handle.expect(":~")
                     linkCheck = main.ONOSbench.handle.before
-                    if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
+                    #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
+                    if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
                         break
                     if i == 39:
                         main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
+                
+                time.sleep(5) #trying to avoid negative values 
 
                 #intents events metrics installed timestamp
                 IEMtimestamps = [0]*(clusterCount + 1)
                 installedTemp = [0]*(clusterCount + 1)
                 for node in range(1, clusterCount +1):
-                    cmd = "onos $OC" + str(node) + " intents-events-metrics|grep Timestamp"
+                    cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
                     raw = ""
-                    while "Timestamp" not in raw:
+                    while "epoch)" not in raw:
                         main.ONOSbench.handle.sendline(cmd)
                         main.ONOSbench.handle.expect(":~")
                         raw = main.ONOSbench.handle.before
 
+                    print(raw)
+
                     intentsTimestamps = {}
                     rawTimestamps = raw.splitlines()
                     for line in rawTimestamps:
@@ -398,13 +391,16 @@
             average = numpy.average(latTemp)
             stdDev = numpy.std(latTemp)
 
+            average = numpy.multiply(average, 1000)
+            stdDev = numpy.multiply(stdDev, 1000)
+
             main.log.report("Scale: " + str(clusterCount) + "  \tIntent batch: " + str(intents))
             main.log.report("Latency average:................" + str(average))
             main.log.report("Latency standard deviation:....." + str(stdDev))
             main.log.report("Mode of last node to respond:..." + str(nodeMode))
             main.log.report("________________________________________________________")
 
-            resultsDB = open("IntentRerouteLatDB", "w+")
+            resultsDB = open("IntentRerouteLatDB", "a")
             resultsDB.write("'" + commit + "',") 
             resultsDB.write(str(clusterCount) + ",")
             resultsDB.write(str(intents) + ",")
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo b/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
index 30a1467..0e45e0f 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
@@ -3,7 +3,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -12,7 +12,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
         </ONOS2cli>
 
         <ONOS3cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
         </ONOS3cli>
 
         <ONOS4cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
         </ONOS4cli>
 
         <ONOS5cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
         </ONOS5cli>
 
         <ONOS6cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
         </ONOS6cli>
 
         <ONOS7cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -75,7 +75,7 @@
         </ONOS7cli>
 
         <ONOS1>
-            <host>10.128.5.51</host>
+            <host>10.254.1.201</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -84,7 +84,7 @@
         </ONOS1>
 
         <ONOS2>
-            <host>10.128.5.52</host>
+            <host>10.254.1.202</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -93,7 +93,7 @@
         </ONOS2>
 
         <ONOS3>
-            <host>10.128.5.53</host>
+            <host>10.254.1.203</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -102,7 +102,7 @@
         </ONOS3>
 
         <ONOS4>
-            <host>10.128.5.54</host>
+            <host>10.254.1.204</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -112,7 +112,7 @@
 
     
         <ONOS5>
-            <host>10.128.5.65</host>
+            <host>10.254.1.205</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -121,7 +121,7 @@
         </ONOS5>
 
         <ONOS6>
-            <host>10.128.5.66</host>
+            <host>10.254.1.206</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -130,7 +130,7 @@
         </ONOS6>
 
         <ONOS7>
-            <host>10.128.5.67</host>
+            <host>10.254.1.207</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
diff --git a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py b/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py
deleted file mode 100644
index 4400262..0000000
--- a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# ScaleOutTemplate
-#
-# CASE1 starts number of nodes specified in param file
-#
-# cameron@onlab.us
-
-import sys
-import os.path
-
-
-class ScaleOutTemplate:
-
-    def __init__( self ):
-        self.default = ''
-
-    def CASE1( self, main ):           
-                                        
-        import time                     
-        global init       
-        try: 
-            if type(init) is not bool: 
-                init = False  
-        except NameError: 
-            init = False 
-       
-        #Load values from params file
-        checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
-        gitPull = main.params[ 'GIT' ][ 'autopull' ]
-        cellName = main.params[ 'ENV' ][ 'cellName' ]
-        Apps = main.params[ 'ENV' ][ 'cellApps' ]
-        BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
-        BENCHUser = main.params[ 'BENCH' ][ 'user' ]
-        MN1Ip = main.params[ 'MN' ][ 'ip1' ]
-        maxNodes = int(main.params[ 'availableNodes' ])
-        skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
-        cellName = main.params[ 'ENV' ][ 'cellName' ]        
-
-        # -- INIT SECTION, ONLY RUNS ONCE -- # 
-        if init == False: 
-            init = True
-            global clusterCount             #number of nodes running
-            global ONOSIp                   #list of ONOS IP addresses
-            global scale 
-            
-            clusterCount = 0
-            ONOSIp = [ 0 ]
-            scale = (main.params[ 'SCALE' ]).split(",")            
-            clusterCount = int(scale[0])
-
-            #Populate ONOSIp with ips from params 
-            for i in range(1, maxNodes + 1): 
-                ipString = 'ip' + str(i) 
-                ONOSIp.append(main.params[ 'CTRL' ][ ipString ])   
-            
-            #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
-            if skipMvn != "yes":
-                mvnResult = main.ONOSbench.cleanInstall()
-
-            #git
-            main.step( "Git checkout and pull " + checkoutBranch )
-            if gitPull == 'on':
-                checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
-                pullResult = main.ONOSbench.gitPull()
-
-            else:
-                checkoutResult = main.TRUE
-                pullResult = main.TRUE
-                main.log.info( "Skipped git checkout and pull" )
-        
-        # -- END OF INIT SECTION --#
-         
-        clusterCount = int(scale[0])
-        scale.remove(scale[0])       
-        
-        #kill off all onos processes 
-        main.log.step("Safety check, killing all ONOS processes")
-        main.log.step("before initiating enviornment setup")
-        for node in range(1, maxNodes + 1):
-            main.ONOSbench.onosDie(ONOSIp[node])
-        
-        #Uninstall everywhere
-        main.log.step( "Cleaning Enviornment..." )
-        for i in range(1, maxNodes + 1):
-            main.log.info(" Uninstalling ONOS " + str(i) )
-            main.ONOSbench.onosUninstall( ONOSIp[i] )
-       
-        #construct the cell file
-        main.log.info("Creating cell file")
-        cellIp = []
-        for node in range (1, clusterCount + 1):
-            cellIp.append(ONOSIp[node])
-
-        main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), *cellIp)
-
-        main.step( "Set Cell" )
-        main.ONOSbench.setCell(cellName)
-        
-        main.step( "Creating ONOS package" )
-        packageResult = main.ONOSbench.onosPackage()  
-
-        main.step( "verify cells" )
-        verifyCellResult = main.ONOSbench.verifyCell()
-      
-        main.log.report( "Initializeing " + str( clusterCount ) + " node cluster." )
-        for node in range(1, clusterCount + 1):
-            main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
-            main.ONOSbench.onosInstall( ONOSIp[node])
-
-        for node in range(1, clusterCount + 1):
-            for i in range( 2 ):
-                isup = main.ONOSbench.isup( ONOSIp[node] )
-                if isup:
-                    main.log.info("ONOS " + str(node) + " is up\n")
-                    break
-            if not isup:
-                main.log.report( "ONOS " + str(node) + " didn't start!" )
-        main.log.info("Startup sequence complete")
-
-    def CASE2( self, main ):
-         
-        print ("clusterCount: " + str(clusterCount)) 
-        print ("scale: " + str(scale)) 
-        print ("ONOSIp: " + str(ONOSIp)) 
-        print ("INIT: " + str(init)) 
-        
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.params b/TestON/tests/SdnIpTest/SdnIpTest.params
index 7f7ef2d..c84af17 100755
--- a/TestON/tests/SdnIpTest/SdnIpTest.params
+++ b/TestON/tests/SdnIpTest/SdnIpTest.params
@@ -1,6 +1,6 @@
 <PARAMS>
     
-    <testcases>4</testcases>
+    <testcases>100, [4]*5</testcases>
 
     #Environment variables
     <ENV>
@@ -8,13 +8,15 @@
     </ENV>
 
     <CTRL>
+    	<numCtrl>1</numCtrl>
         <ip1>10.128.4.52</ip1>
         <port1>6633</port1>
     </CTRL>
 
     <GIT>
-        <autoPull>off</autoPull>
-        <checkout>master</checkout>
+        <autoPull>on</autoPull>
+        <branch1>master</branch1>
+        <branch2>onos-1.2</branch2>
     </GIT>
 
     <JSON>
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.py b/TestON/tests/SdnIpTest/SdnIpTest.py
index 85ac21e..43bf580 100644
--- a/TestON/tests/SdnIpTest/SdnIpTest.py
+++ b/TestON/tests/SdnIpTest/SdnIpTest.py
@@ -1,22 +1,86 @@
-# from cupshelpers.config import prefix
 
-# Testing the basic functionality of SDN-IP
-
-
+# Testing the functionality of SDN-IP with single ONOS instance
 class SdnIpTest:
 
     def __init__( self ):
         self.default = ''
+        global branchName
 
-# from cupshelpers.config import prefix
+    # This case is to setup ONOS
+    def CASE100( self, main ):
+        """
+           CASE100 is to compile ONOS and push it to the test machines
+           Startup sequence:
+           git pull
+           mvn clean install
+           onos-package
+           cell <name>
+           onos-verify-cell
+           onos-install -f
+           onos-wait-for-start
+        """
+        import time
+        main.case( "Setting up test environment" )
 
-# Testing the basic functionality of SDN-IP
+        cellName = main.params[ 'ENV' ][ 'cellName' ]
+        ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
 
+        main.step( "Applying cell variable to environment" )
+        cellResult = main.ONOSbench.setCell( cellName )
+        verifyResult = main.ONOSbench.verifyCell()
 
-class SdnIpTest:
+        branchName = main.ONOSbench.getBranchName()
+        main.log.info( "ONOS is on branch: " + branchName )
 
-    def __init__( self ):
-        self.default = ''
+        main.log.report( "Uninstalling ONOS" )
+        main.ONOSbench.onosUninstall( ONOS1Ip )
+
+        cleanInstallResult = main.TRUE
+        gitPullResult = main.TRUE
+
+        main.step( "Git pull" )
+        gitPullResult = main.ONOSbench.gitPull()
+
+        main.step( "Using mvn clean & install" )
+        cleanInstallResult = main.TRUE
+#         if gitPullResult == main.TRUE:
+#             cleanInstallResult = main.ONOSbench.cleanInstall()
+#         else:
+#             main.log.warn( "Did not pull new code so skipping mvn " +
+#                             "clean install" )
+        cleanInstallResult = main.ONOSbench.cleanInstall( mciTimeout= 1000 )
+        main.ONOSbench.getVersion( report=True )
+
+        main.step( "Creating ONOS package" )
+        packageResult = main.ONOSbench.onosPackage( opTimeout=500 )
+
+        main.step( "Installing ONOS package" )
+        onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
+                                                           node=ONOS1Ip )
+
+        main.step( "Checking if ONOS is up yet" )
+        for i in range( 2 ):
+            onos1Isup = main.ONOSbench.isup( ONOS1Ip, timeout=420 )
+            if onos1Isup:
+                break
+        if not onos1Isup:
+            main.log.report( "ONOS1 didn't start!" )
+
+        cliResult = main.ONOScli.startOnosCli( ONOS1Ip,
+                commandlineTimeout=100, onosStartTimeout=600)
+
+        case1Result = ( cleanInstallResult and packageResult and
+                        cellResult and verifyResult and
+                        onos1InstallResult and
+                        onos1Isup and cliResult )
+
+        utilities.assert_equals( expect=main.TRUE, actual=case1Result,
+                                 onpass="ONOS startup successful",
+                                 onfail="ONOS startup NOT successful" )
+
+        if case1Result == main.FALSE:
+            main.cleanup()
+            main.exit()
 
     def CASE4( self, main ):
         """
@@ -33,14 +97,12 @@
         import time
         import json
         from operator import eq
-        # from datetime import datetime
         from time import localtime, strftime
 
-        main.case("The test case is to help to setup the TestON environment \
-            and test new drivers" )
-        # SDNIPJSONFILEPATH = "../tests/SdnIpTest/sdnip.json"
+        main.case("This case is to testing the functionality of SDN-IP with \
+        single ONOS instance" )
         SDNIPJSONFILEPATH = \
-            "/home/admin/workspace/onos/tools/package/config/sdnip.json"
+            "/home/admin/ONOS/tools/package/config/sdnip.json"
         # all expected routes for all BGP peers
         allRoutesExpected = []
         main.step( "Start to generate routes for all BGP peers" )
@@ -79,37 +141,12 @@
         routeIntentsExpected = routeIntentsExpectedHost3 + \
             routeIntentsExpectedHost4 + routeIntentsExpectedHost5
 
-        cellName = main.params[ 'ENV' ][ 'cellName' ]
-        ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
-        main.step( "Set cell for ONOS-cli environment" )
-        main.ONOScli.setCell( cellName )
-        verifyResult = main.ONOSbench.verifyCell()
-
-        main.log.report( "Removing raft logs" )
-        main.ONOSbench.onosRemoveRaftLogs()
-        main.log.report( "Uninstalling ONOS" )
-        main.ONOSbench.onosUninstall( ONOS1Ip )
-
-        main.step( "Installing ONOS package" )
-        onos1InstallResult = main.ONOSbench.onosInstall(
-            options="-f", node=ONOS1Ip )
-
-        main.step( "Checking if ONOS is up yet" )
-        time.sleep( 150 )
-        onos1Isup = main.ONOSbench.isup( ONOS1Ip )
-        if not onos1Isup:
-            main.log.report( "ONOS1 didn't start!" )
-
-        main.step( "Start ONOS-cli" )
-
-        main.ONOScli.startOnosCli( ONOS1Ip )
-
         main.step( "Get devices in the network" )
         listResult = main.ONOScli.devices( jsonFormat=False )
         main.log.info( listResult )
         time.sleep( 10 )
-        main.log.info( "Installing sdn-ip feature" )
-        main.ONOScli.featureInstall( "onos-app-sdnip" )
+        main.log.info( "Activate sdn-ip application" )
+        main.ONOScli.activateApp( "org.onosproject.sdnip" )
         time.sleep( 10 )
         main.step( "Login all BGP peers and add routes into peers" )
 
@@ -138,8 +175,8 @@
             prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
             main.log.info( prefixesHostX )
             for prefix in prefixesHostX:
-                allRoutesExpected.append(
-                    prefix + "/" + "192.168.40." + str( i - 100 ) )
+                allRoutesExpected.append( prefix + "/" + "192.168.40."
+                                           + str( i - 100 ) )
 
             routeIntentsExpectedHostX = \
             main.QuaggaCliHost.generateExpectedOnePeerRouteIntents(
@@ -158,13 +195,11 @@
             QuaggaCliHostX.addRoutes( prefixesHostX, 1 )
 
         time.sleep( 60 )
-
         # get routes inside SDN-IP
         getRoutesResult = main.ONOScli.routes( jsonFormat=True )
 
-        # parse routes from ONOS CLI
         allRoutesActual = \
-            main.QuaggaCliHost3.extractActualRoutes( getRoutesResult )
+            main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
 
         allRoutesStrExpected = str( sorted( allRoutesExpected ) )
         allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
@@ -189,7 +224,281 @@
 
         main.step( "Check MultiPointToSinglePointIntent intents installed" )
         # routeIntentsExpected are generated when generating routes
-        # get rpoute intents from ONOS CLI
+        # get route intents from ONOS CLI
+        routeIntentsActualNum = \
+            main.QuaggaCliHost3.extractActualRouteIntentNum( getIntentsResult )
+        routeIntentsExpectedNum = 1030
+        main.log.info( "MultiPointToSinglePoint Intent Num expected is:" )
+        main.log.info( routeIntentsExpectedNum )
+        main.log.info( "MultiPointToSinglePoint Intent NUM Actual is:" )
+        main.log.info( routeIntentsActualNum )
+        utilities.assertEquals(
+            expect=True,
+            actual=eq( routeIntentsExpectedNum, routeIntentsActualNum ),
+            onpass="***MultiPointToSinglePoint Intent Num in SDN-IP is \
+            correct!***",
+            onfail="***MultiPointToSinglePoint Intent Num in SDN-IP is \
+            wrong!***" )
+
+        if( eq( routeIntentsExpectedNum, routeIntentsActualNum ) ):
+            main.log.report( "***MultiPointToSinglePoint Intents before \
+            deleting routes correct!***" )
+        else:
+            main.log.report( "***MultiPointToSinglePoint Intents before \
+            deleting routes wrong!***" )
+
+        main.step( "Check BGP PointToPointIntent intents installed" )
+
+        bgpIntentsActualNum = \
+            main.QuaggaCliHost3.extractActualBgpIntentNum( getIntentsResult )
+        bgpIntentsExpectedNum = 624
+        main.log.info( "bgpIntentsExpected num is:" )
+        main.log.info( bgpIntentsExpectedNum )
+        main.log.info( "bgpIntentsActual num is:" )
+        main.log.info( bgpIntentsActualNum)
+        utilities.assertEquals(
+            expect=True,
+            actual=eq( bgpIntentsExpectedNum, bgpIntentsActualNum ),
+            onpass="***PointToPointIntent Intent Num in SDN-IP are correct!***",
+            onfail="***PointToPointIntent Intent Num in SDN-IP are wrong!***" )
+        if ( eq( bgpIntentsExpectedNum, bgpIntentsActualNum ) ):
+            main.log.report(
+                "***PointToPointIntent Intents in SDN-IP are correct!***" )
+        else:
+            main.log.report(
+                "***PointToPointIntent Intents in SDN-IP are wrong!***" )
+
+        #============================= Ping Test ========================
+        # Wait until all MultiPointToSinglePoint intents are in system
+        time.sleep( 20 )
+        pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
+        pingTestResultsFile = \
+        "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-before-delete-routes-"\
+            + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
+        pingTestResults = main.QuaggaCliHost.pingTest(
+            "1.168.30.100", pingTestScript, pingTestResultsFile )
+        main.log.info( pingTestResults )
+        time.sleep( 20 )
+
+        #============================= Deleting Routes ==================
+        main.step( "Check deleting routes installed" )
+        main.QuaggaCliHost3.deleteRoutes( prefixesHost3, 1 )
+        main.QuaggaCliHost4.deleteRoutes( prefixesHost4, 1 )
+        main.QuaggaCliHost5.deleteRoutes( prefixesHost5, 1 )
+
+        for i in range( 101, 201 ):
+            prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
+            main.log.info( prefixesHostX )
+            QuaggaCliHostX = getattr( main, ( 'QuaggaCliHost' + str( i ) ) )
+            QuaggaCliHostX.deleteRoutes( prefixesHostX, 1 )
+
+        getRoutesResult = main.ONOScli.routes( jsonFormat=True )
+        allRoutesActual = \
+            main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+        main.log.info( "allRoutes_actual = " )
+        main.log.info( allRoutesActual )
+
+        utilities.assertEquals(
+            expect="[]", actual=str( allRoutesActual ),
+            onpass="***Route number in SDN-IP is 0, correct!***",
+            onfail="***Routes number in SDN-IP is not 0, wrong!***" )
+
+        if( eq( allRoutesStrExpected, allRoutesStrActual ) ):
+            main.log.report( "***Routes in SDN-IP after deleting correct!***" )
+        else:
+            main.log.report( "***Routes in SDN-IP after deleting wrong!***" )
+
+        main.step( "Check intents after deleting routes" )
+        getIntentsResult = main.ONOScli.intents( jsonFormat=True )
+        routeIntentsActualNum = \
+            main.QuaggaCliHost3.extractActualRouteIntentNum(
+                getIntentsResult )
+        main.log.info( "route Intents Actual Num is: " )
+        main.log.info( routeIntentsActualNum )
+        utilities.assertEquals(
+            expect=0, actual=routeIntentsActualNum,
+            onpass="***MultiPointToSinglePoint Intent Num in SDN-IP is 0, \
+            correct!***",
+            onfail="***MultiPointToSinglePoint Intent Num in SDN-IP is not 0, \
+            wrong!***" )
+
+        if( eq( 0, routeIntentsActualNum ) ):
+            main.log.report( "***MultiPointToSinglePoint Intents after \
+            deleting routes correct!***" )
+        else:
+            main.log.report( "***MultiPointToSinglePoint Intents after \
+            deleting routes wrong!***" )
+
+        time.sleep( 20 )
+        pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
+        pingTestResultsFile = \
+        "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-"\
+            + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
+        pingTestResults = main.QuaggaCliHost.pingTest(
+            "1.168.30.100", pingTestScript, pingTestResultsFile )
+        main.log.info( pingTestResults )
+        time.sleep( 100 )
+
+    def CASE_4_origin( self, main ):
+        """
+        Test the SDN-IP functionality
+        allRoutesExpected: all expected routes for all BGP peers
+        routeIntentsExpected: all expected MultiPointToSinglePointIntent \
+        intents
+        bgpIntentsExpected: expected PointToPointIntent intents
+        allRoutesActual: all routes from ONOS LCI
+        routeIntentsActual: actual MultiPointToSinglePointIntent intents from \
+        ONOS CLI
+        bgpIntentsActual: actual PointToPointIntent intents from ONOS CLI
+        """
+        import time
+        import json
+        from operator import eq
+        from time import localtime, strftime
+
+        main.case("This case is to testing the functionality of SDN-IP with \
+        single ONOS instance" )
+        SDNIPJSONFILEPATH = \
+            "/home/admin/ONOS/tools/package/config/sdnip.json"
+        # all expected routes for all BGP peers
+        allRoutesExpected = []
+        main.step( "Start to generate routes for all BGP peers" )
+        main.log.info( "Generate prefixes for host3" )
+        prefixesHost3 = main.QuaggaCliHost3.generatePrefixes( 3, 10 )
+        main.log.info( prefixesHost3 )
+        # generate route with next hop
+        for prefix in prefixesHost3:
+            allRoutesExpected.append( prefix + "/" + "192.168.20.1" )
+        routeIntentsExpectedHost3 = \
+            main.QuaggaCliHost3.generateExpectedOnePeerRouteIntents(
+            prefixesHost3, "192.168.20.1", "00:00:00:00:02:02",
+            SDNIPJSONFILEPATH )
+
+        main.log.info( "Generate prefixes for host4" )
+        prefixesHost4 = main.QuaggaCliHost4.generatePrefixes( 4, 10 )
+        main.log.info( prefixesHost4 )
+        # generate route with next hop
+        for prefix in prefixesHost4:
+            allRoutesExpected.append( prefix + "/" + "192.168.30.1" )
+        routeIntentsExpectedHost4 = \
+            main.QuaggaCliHost4.generateExpectedOnePeerRouteIntents(
+            prefixesHost4, "192.168.30.1", "00:00:00:00:03:01",
+            SDNIPJSONFILEPATH )
+
+        main.log.info( "Generate prefixes for host5" )
+        prefixesHost5 = main.QuaggaCliHost5.generatePrefixes( 5, 10 )
+        main.log.info( prefixesHost5 )
+        for prefix in prefixesHost5:
+            allRoutesExpected.append( prefix + "/" + "192.168.60.2" )
+        routeIntentsExpectedHost5 = \
+            main.QuaggaCliHost5.generateExpectedOnePeerRouteIntents(
+            prefixesHost5, "192.168.60.1", "00:00:00:00:06:02",
+            SDNIPJSONFILEPATH )
+
+        routeIntentsExpected = routeIntentsExpectedHost3 + \
+            routeIntentsExpectedHost4 + routeIntentsExpectedHost5
+
+        main.step( "Get devices in the network" )
+        listResult = main.ONOScli.devices( jsonFormat=False )
+        main.log.info( listResult )
+        time.sleep( 10 )
+        if branchName == "onos-1.1":
+            main.log.info( "Installing sdn-ip feature" )
+            main.ONOScli.featureInstall( "onos-app-sdnip" )
+        else:
+            main.log.info( "Activate sdn-ip application" )
+            main.ONOScli.activateApp( "org.onosproject.sdnip" )
+        #main.log.info( "Installing sdn-ip feature" )
+        #main.ONOScli.featureInstall( "onos-app-sdnip" )
+        time.sleep( 10 )
+        main.step( "Login all BGP peers and add routes into peers" )
+
+        main.log.info( "Login Quagga CLI on host3" )
+        main.QuaggaCliHost3.loginQuagga( "1.168.30.2" )
+        main.log.info( "Enter configuration model of Quagga CLI on host3" )
+        main.QuaggaCliHost3.enterConfig( 64514 )
+        main.log.info( "Add routes to Quagga on host3" )
+        main.QuaggaCliHost3.addRoutes( prefixesHost3, 1 )
+
+        main.log.info( "Login Quagga CLI on host4" )
+        main.QuaggaCliHost4.loginQuagga( "1.168.30.3" )
+        main.log.info( "Enter configuration model of Quagga CLI on host4" )
+        main.QuaggaCliHost4.enterConfig( 64516 )
+        main.log.info( "Add routes to Quagga on host4" )
+        main.QuaggaCliHost4.addRoutes( prefixesHost4, 1 )
+
+        main.log.info( "Login Quagga CLI on host5" )
+        main.QuaggaCliHost5.loginQuagga( "1.168.30.5" )
+        main.log.info( "Enter configuration model of Quagga CLI on host5" )
+        main.QuaggaCliHost5.enterConfig( 64521 )
+        main.log.info( "Add routes to Quagga on host5" )
+        main.QuaggaCliHost5.addRoutes( prefixesHost5, 1 )
+
+        for i in range( 101, 201 ):
+            prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
+            main.log.info( prefixesHostX )
+            for prefix in prefixesHostX:
+                allRoutesExpected.append( prefix + "/" + "192.168.40."
+                                           + str( i - 100 ) )
+
+            routeIntentsExpectedHostX = \
+            main.QuaggaCliHost.generateExpectedOnePeerRouteIntents(
+                prefixesHostX, "192.168.40." + str( i - 100 ),
+                "00:00:%02d:00:00:90" % ( i - 101 ), SDNIPJSONFILEPATH )
+            routeIntentsExpected = routeIntentsExpected + \
+                routeIntentsExpectedHostX
+
+            main.log.info( "Login Quagga CLI on host" + str( i ) )
+            QuaggaCliHostX = getattr( main, ( 'QuaggaCliHost' + str( i ) ) )
+            QuaggaCliHostX.loginQuagga( "1.168.30." + str( i ) )
+            main.log.info(
+                "Enter configuration model of Quagga CLI on host" + str( i ) )
+            QuaggaCliHostX.enterConfig( 65000 + i - 100 )
+            main.log.info( "Add routes to Quagga on host" + str( i ) )
+            QuaggaCliHostX.addRoutes( prefixesHostX, 1 )
+
+        time.sleep( 60 )
+        # get routes inside SDN-IP
+        getRoutesResult = main.ONOScli.routes( jsonFormat=True )
+
+        # parse routes from ONOS CLI
+        #if branchName == "master":
+        #    allRoutesActual = \
+        #    main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+        #elif branchName == "onos-1.0":
+        #    allRoutesActual = \
+        #    main.QuaggaCliHost3.extractActualRoutesOneDotZero( getRoutesResult )
+        #else:
+        #    main.log("ONOS is on wrong branch")
+        #    exit
+
+        allRoutesActual = \
+            main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+
+        allRoutesStrExpected = str( sorted( allRoutesExpected ) )
+        allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
+        main.step( "Check routes installed" )
+        main.log.info( "Routes expected:" )
+        main.log.info( allRoutesStrExpected )
+        main.log.info( "Routes get from ONOS CLI:" )
+        main.log.info( allRoutesStrActual )
+        utilities.assertEquals(
+            expect=allRoutesStrExpected, actual=allRoutesStrActual,
+            onpass="***Routes in SDN-IP are correct!***",
+            onfail="***Routes in SDN-IP are wrong!***" )
+        if( eq( allRoutesStrExpected, allRoutesStrActual ) ):
+            main.log.report(
+                "***Routes in SDN-IP after adding routes are correct!***" )
+        else:
+            main.log.report(
+                "***Routes in SDN-IP after adding routes are wrong!***" )
+
+        time.sleep( 20 )
+        getIntentsResult = main.ONOScli.intents( jsonFormat=True )
+
+        main.step( "Check MultiPointToSinglePointIntent intents installed" )
+        # routeIntentsExpected are generated when generating routes
+        # get route intents from ONOS CLI
         routeIntentsActual = \
             main.QuaggaCliHost3.extractActualRouteIntents(
                 getIntentsResult )
@@ -243,7 +552,7 @@
                 "***PointToPointIntent Intents in SDN-IP are wrong!***" )
 
         #============================= Ping Test ========================
-        # wait until all MultiPointToSinglePoint
+        # Wait until all MultiPointToSinglePoint intents are in system
         time.sleep( 20 )
         pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
         pingTestResultsFile = \
@@ -306,22 +615,19 @@
         time.sleep( 20 )
         pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
         pingTestResultsFile = \
-        "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-" \
+        "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-"\
             + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
         pingTestResults = main.QuaggaCliHost.pingTest(
             "1.168.30.100", pingTestScript, pingTestResultsFile )
         main.log.info( pingTestResults )
         time.sleep( 100 )
 
-        # main.step( "Test whether Mininet is started" )
-        # main.Mininet2.handle.sendline( "xterm host1" )
-        # main.Mininet2.handle.expect( "mininet>" )
-
     def CASE3( self, main ):
         """
         Test the SDN-IP functionality
         allRoutesExpected: all expected routes for all BGP peers
-        routeIntentsExpected: all expected MultiPointToSinglePointIntent intents
+        routeIntentsExpected: all expected MultiPointToSinglePointIntent \
+        intents
         bgpIntentsExpected: expected PointToPointIntent intents
         allRoutesActual: all routes from ONOS LCI
         routeIntentsActual: actual MultiPointToSinglePointIntent intents from \
@@ -338,7 +644,7 @@
             environment and test new drivers" )
         # SDNIPJSONFILEPATH = "../tests/SdnIpTest/sdnip.json"
         SDNIPJSONFILEPATH = \
-            "/home/admin/workspace/onos/tools/package/config/sdnip.json"
+            "/home/admin/ONOS/tools/package/config/sdnip.json"
         # all expected routes for all BGP peers
         allRoutesExpected = []
         main.step( "Start to generate routes for all BGP peers" )
@@ -396,8 +702,12 @@
         listResult = main.ONOScli.devices( jsonFormat=False )
         main.log.info( listResult )
         time.sleep( 10 )
-        main.log.info( "Installing sdn-ip feature" )
-        main.ONOScli.featureInstall( "onos-app-sdnip" )
+        if branchName == "onos-1.1":
+            main.log.info( "Installing sdn-ip feature" )
+            main.ONOScli.featureInstall( "onos-app-sdnip" )
+        else:
+            main.log.info( "Activate sdn-ip application" )
+            main.ONOScli.activateApp( "org.onosproject.sdnip" )
         time.sleep( 10 )
         main.step( "Login all BGP peers and add routes into peers" )
 
diff --git a/TestON/tests/TopoPerfNext/Backup/TopoPerfNext.py b/TestON/tests/TopoPerfNext/Backup/TopoPerfNext.py
deleted file mode 100644
index cc40b94..0000000
--- a/TestON/tests/TopoPerfNext/Backup/TopoPerfNext.py
+++ /dev/null
@@ -1,1710 +0,0 @@
-#TopoPerfNext
-#
-#Topology Performance test for ONOS-next
-#
-#andrew@onlab.us
-#
-#If your machine does not come with numpy
-#run the following command:
-#sudo apt-get install python-numpy python-scipy 
-
-import time
-import sys
-import os
-import re
-
-class TopoPerfNext:
-    def __init__(self):
-        self.default = ''
-
-    def CASE1(self, main):
-        '''
-        ONOS startup sequence
-        '''
-        import time
-        
-        ## Global cluster count for scale-out purposes
-        global cluster_count
-        #Set initial cluster count
-        cluster_count = 1 
-        ##
-
-        cell_name = main.params['ENV']['cellName']
-
-        git_pull = main.params['GIT']['autoPull']
-        checkout_branch = main.params['GIT']['checkout']
-
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        
-        #### Hardcoded ONOS nodes particular to my env ####
-        ONOS4_ip = "10.128.174.4"
-        ONOS5_ip = "10.128.174.5"
-        ONOS6_ip = "10.128.174.6"
-        ONOS7_ip = "10.128.174.7"
-        #### ####
-
-        MN1_ip = main.params['MN']['ip1']
-        BENCH_ip = main.params['BENCH']['ip']
-
-        topo_cfg_file = main.params['TEST']['topo_config_file']
-        topo_cfg_name = main.params['TEST']['topo_config_name']
-        
-        main.case("Setting up test environment")
-        main.log.info("Copying topology event accumulator config"+\
-            " to ONOS /package/etc")
-        main.ONOSbench.handle.sendline("cp ~/"+\
-            topo_cfg_file+\
-            " ~/ONOS/tools/package/etc/"+\
-            topo_cfg_name)
-        main.ONOSbench.handle.expect("\$")
-
-        main.log.report("Setting up test environment")
-
-        main.step("Cleaning previously installed ONOS if any")
-        main.ONOSbench.onos_uninstall(node_ip=ONOS2_ip)
-        main.ONOSbench.onos_uninstall(node_ip=ONOS3_ip)
-        main.ONOSbench.onos_uninstall(node_ip=ONOS4_ip)
-        main.ONOSbench.onos_uninstall(node_ip=ONOS5_ip)
-        main.ONOSbench.onos_uninstall(node_ip=ONOS6_ip)
-        main.ONOSbench.onos_uninstall(node_ip=ONOS7_ip)
-
-        main.step("Creating cell file")
-        cell_file_result = main.ONOSbench.create_cell_file(
-                BENCH_ip, cell_name, MN1_ip, "onos-core,onos-app-metrics",
-                ONOS1_ip)
-
-        main.step("Applying cell file to environment")
-        cell_apply_result = main.ONOSbench.set_cell(cell_name)
-        verify_cell_result = main.ONOSbench.verify_cell()
-        
-        #NOTE: This step may be removed after proper 
-        #      copy cat log functionality
-        main.step("Removing raft/copy-cat logs from ONOS nodes")
-        main.ONOSbench.onos_remove_raft_logs()
-        time.sleep(30)
-
-        main.step("Git checkout and pull "+checkout_branch)
-        if git_pull == 'on':
-            checkout_result = \
-                    main.ONOSbench.git_checkout(checkout_branch)
-            pull_result = main.ONOSbench.git_pull()
-        else:
-            checkout_result = main.TRUE
-            pull_result = main.TRUE
-            main.log.info("Skipped git checkout and pull")
-
-        main.log.report("Commit information - ")
-        main.ONOSbench.get_version(report=True)
-
-        main.step("Using mvn clean & install")
-        mvn_result = main.ONOSbench.clean_install()
-        mvn_result = main.TRUE
-
-        main.step("Set cell for ONOS cli env")
-        main.ONOS1cli.set_cell(cell_name)
-        #main.ONOS2cli.set_cell(cell_name)
-        #main.ONOS3cli.set_cell(cell_name)
-
-        main.step("Creating ONOS package")
-        package_result = main.ONOSbench.onos_package()
-
-        main.step("Installing ONOS package")
-        install1_result = main.ONOSbench.onos_install(node=ONOS1_ip)
-        #install2_result = main.ONOSbench.onos_install(node=ONOS2_ip)
-        #install3_result = main.ONOSbench.onos_install(node=ONOS3_ip)
-
-        time.sleep(10)
-
-        main.step("Start onos cli")
-        cli1 = main.ONOS1cli.start_onos_cli(ONOS1_ip)
-        #cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
-        #cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)
-
-        utilities.assert_equals(expect=main.TRUE,
-                actual= cell_file_result and cell_apply_result and\
-                        verify_cell_result and checkout_result and\
-                        pull_result and mvn_result and\
-                        install1_result, #and install2_result and\
-                        #install3_result,
-                onpass="Test Environment setup successful",
-                onfail="Failed to setup test environment")
-
-    def CASE2(self, main):
-        '''
-        Assign s1 to ONOS1 and measure latency
-        
-        There are 4 levels of latency measurements to this test:
-        1) End-to-end measurement: Complete end-to-end measurement
-           from TCP (SYN/ACK) handshake to Graph change
-        2) OFP-to-graph measurement: 'ONOS processing' snippet of
-           measurement from OFP Vendor message to Graph change
-        3) OFP-to-device measurement: 'ONOS processing without 
-           graph change' snippet of measurement from OFP vendor
-           message to Device change timestamp
-        4) T0-to-device measurement: Measurement that includes
-           the switch handshake to devices timestamp without 
-           the graph view change. (TCP handshake -> Device 
-           change)
-        '''
-        import time
-        import subprocess
-        import json
-        import requests
-        import os
-        import numpy
-        global cluster_count
-
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS4_ip = main.params['CTRL']['ip4']
-        ONOS5_ip = main.params['CTRL']['ip5']
-        ONOS6_ip = main.params['CTRL']['ip6']
-        ONOS7_ip = main.params['CTRL']['ip7']
-
-        ONOS_user = main.params['CTRL']['user']
-
-        default_sw_port = main.params['CTRL']['port1']
-       
-        #Number of iterations of case
-        num_iter = main.params['TEST']['numIter']
-        #Number of first 'x' iterations to ignore:
-        iter_ignore = int(main.params['TEST']['iterIgnore'])
-
-        #Timestamp 'keys' for json metrics output.
-        #These are subject to change, hence moved into params
-        deviceTimestamp = main.params['JSON']['deviceTimestamp']
-        graphTimestamp = main.params['JSON']['graphTimestamp']
-
-        debug_mode = main.params['TEST']['debugMode']
-        onos_log = main.params['TEST']['onosLogFile']
-
-        #Threshold for the test
-        threshold_str = main.params['TEST']['singleSwThreshold']
-        threshold_obj = threshold_str.split(",")
-        threshold_min = int(threshold_obj[0])
-        threshold_max = int(threshold_obj[1])
-
-        #List of switch add latency collected from
-        #all iterations
-        latency_end_to_end_list = []
-        latency_ofp_to_graph_list = []
-        latency_ofp_to_device_list = []
-        latency_t0_to_device_list = []
-        latency_tcp_to_ofp_list = []
-
-        #Directory/file to store tshark results
-        tshark_of_output = "/tmp/tshark_of_topo.txt"
-        tshark_tcp_output = "/tmp/tshark_tcp_topo.txt"
-
-        #String to grep in tshark output
-        tshark_tcp_string = "TCP 74 "+default_sw_port
-        tshark_of_string = "OFP 86 Vendor"
-     
-        #Initialize assertion to TRUE
-        assertion = main.TRUE
-      
-        local_time = time.strftime('%x %X')
-        local_time = local_time.replace("/","")
-        local_time = local_time.replace(" ","_")
-        local_time = local_time.replace(":","")
-        if debug_mode == 'on':
-            main.ONOS1.tshark_pcap("eth0",
-                    "/tmp/single_sw_lat_pcap_"+local_time) 
-
-            main.log.info("Debug mode is on")
-
-        main.log.report("Latency of adding one switch to controller")
-        main.log.report("First "+str(iter_ignore)+" iterations ignored"+
-                " for jvm warmup time")
-        main.log.report("Total iterations of test: "+str(num_iter))
-
-        for i in range(0, int(num_iter)):
-            main.log.info("Starting tshark capture")
-
-            #* TCP [ACK, SYN] is used as t0_a, the
-            #  very first "exchange" between ONOS and 
-            #  the switch for end-to-end measurement
-            #* OFP [Stats Reply] is used for t0_b
-            #  the very last OFP message between ONOS
-            #  and the switch for ONOS measurement
-            main.ONOS1.tshark_grep(tshark_tcp_string,
-                    tshark_tcp_output)
-            main.ONOS1.tshark_grep(tshark_of_string,
-                    tshark_of_output)
-
-            #Wait and ensure tshark is started and 
-            #capturing
-            time.sleep(10)
-
-            main.log.info("Assigning s1 to controller")
-
-            main.Mininet1.assign_sw_controller(sw="1",
-                    ip1=ONOS1_ip, port1=default_sw_port)
-
-            #Wait and ensure switch is assigned
-            #before stopping tshark
-            time.sleep(30)
-   
-            main.log.info("Stopping all Tshark processes")
-            main.ONOS1.stop_tshark()
-
-            #tshark output is saved in ONOS. Use subprocess
-            #to copy over files to TestON for parsing
-            main.log.info("Copying over tshark files")
-            
-            #TCP CAPTURE ****
-            #Copy the tshark output from ONOS machine to
-            #TestON machine in tshark_tcp_output directory>file
-            os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
-                    tshark_tcp_output+" /tmp/") 
-            tcp_file = open(tshark_tcp_output, 'r')
-            temp_text = tcp_file.readline()
-            temp_text = temp_text.split(" ")
-
-            main.log.info("Object read in from TCP capture: "+
-                    str(temp_text))
-            if len(temp_text) > 1:
-                t0_tcp = float(temp_text[1])*1000.0
-            else:
-                main.log.error("Tshark output file for TCP"+
-                        " returned unexpected results")
-                t0_tcp = 0
-                assertion = main.FALSE
-            
-            tcp_file.close()
-            #****************
-
-            #OF CAPTURE ****
-            os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
-                    tshark_of_output+" /tmp/")
-            of_file = open(tshark_of_output, 'r')
-           
-            line_ofp = ""
-            #Read until last line of file
-            while True:
-                temp_text = of_file.readline()
-                if temp_text !='':
-                    line_ofp = temp_text
-                else:
-                    break 
-            obj = line_ofp.split(" ")
-            
-            main.log.info("Object read in from OFP capture: "+
-                    str(line_ofp))
-    
-            if len(line_ofp) > 1:
-                t0_ofp = float(obj[1])*1000.0
-            else:
-                main.log.error("Tshark output file for OFP"+
-                        " returned unexpected results")
-                t0_ofp = 0
-                assertion = main.FALSE
-            
-            of_file.close()
-            #****************
-           
-            json_str_1 = main.ONOS1cli.topology_events_metrics()
-            #Initialize scale-out variables 
-            json_str_2 = "" 
-            json_str_3 = ""
-            json_str_4 = ""
-            json_str_5 = ""
-            json_str_6 = ""
-            json_str_7 = ""
-
-            json_obj_1 = json.loads(json_str_1)
-            json_obj_2 = json.loads(json_str_2)
-            json_obj_3 = json.loads(json_str_3)
-            #Initialize scale-out variables
-            json_obj_4 = ""
-            json_obj_5 = ""
-            json_obj_6 = ""
-            json_obj_7 = ""
-
-            #Include scale-out measurements when applicable
-            if cluster_count == 5:
-                json_str_4 = main.ONOS4cli.topology_events_metrics()
-                json_str_5 = main.ONOS5cli.topology_events_metrics()
-                
-                json_obj_4 = json.loads(json_str_4)
-                json_obj_5 = json.loads(json_str_5)
-            elif cluster_count == 6:
-                main.log.info("TODO: create even number cluster events")
-            elif cluster_count == 7:
-                json_str_6 = main.ONOS6cli.topology_events_metrics()
-                json_str_7 = main.ONOS7cli.topology_events_metrics()
-
-                json_obj_6 = json.loads(json_str_6)
-                json_obj_7 = json.loads(json_str_7)
-
-            #Obtain graph timestamp. This timestsamp captures
-            #the epoch time at which the topology graph was updated.
-            graph_timestamp_1 = \
-                    json_obj_1[graphTimestamp]['value']
-            graph_timestamp_2 = \
-                    json_obj_2[graphTimestamp]['value']
-            graph_timestamp_3 = \
-                    json_obj_3[graphTimestamp]['value']
-
-            #Obtain device timestamp. This timestamp captures
-            #the epoch time at which the device event happened
-            device_timestamp_1 = \
-                    json_obj_1[deviceTimestamp]['value'] 
-            device_timestamp_2 = \
-                    json_obj_2[deviceTimestamp]['value'] 
-            device_timestamp_3 = \
-                    json_obj_3[deviceTimestamp]['value'] 
-
-            #t0 to device processing latency 
-            delta_device_1 = int(device_timestamp_1) - int(t0_tcp)
-            delta_device_2 = int(device_timestamp_2) - int(t0_tcp)
-            delta_device_3 = int(device_timestamp_3) - int(t0_tcp)
-        
-            #Get average of delta from all instances
-            avg_delta_device = \
-                    (int(delta_device_1)+\
-                     int(delta_device_2)+\
-                     int(delta_device_3)) / 3
-
-            #Ensure avg delta meets the threshold before appending
-            if avg_delta_device > 0.0 and avg_delta_device < 10000\
-                    and int(i) > iter_ignore:
-                latency_t0_to_device_list.append(avg_delta_device)
-            else:
-                main.log.info("Results for t0-to-device ignored"+\
-                        "due to excess in threshold / warmup iteration.")
-
-            #t0 to graph processing latency (end-to-end)
-            delta_graph_1 = int(graph_timestamp_1) - int(t0_tcp)
-            delta_graph_2 = int(graph_timestamp_2) - int(t0_tcp)
-            delta_graph_3 = int(graph_timestamp_3) - int(t0_tcp)
-        
-            #Get average of delta from all instances
-            #TODO: use max delta graph
-            #max_delta_graph = max(three)
-            avg_delta_graph = \
-                    (int(delta_graph_1)+\
-                     int(delta_graph_2)+\
-                     int(delta_graph_3)) / 3
-
-            #Ensure avg delta meets the threshold before appending
-            if avg_delta_graph > 0.0 and avg_delta_graph < 10000\
-                    and int(i) > iter_ignore:
-                latency_end_to_end_list.append(avg_delta_graph)
-            else:
-                main.log.info("Results for end-to-end ignored"+\
-                        "due to excess in threshold")
-
-            #ofp to graph processing latency (ONOS processing)
-            delta_ofp_graph_1 = int(graph_timestamp_1) - int(t0_ofp)
-            delta_ofp_graph_2 = int(graph_timestamp_2) - int(t0_ofp)
-            delta_ofp_graph_3 = int(graph_timestamp_3) - int(t0_ofp)
-            
-            avg_delta_ofp_graph = \
-                    (int(delta_ofp_graph_1)+\
-                     int(delta_ofp_graph_2)+\
-                     int(delta_ofp_graph_3)) / 3
-            
-            if avg_delta_ofp_graph > threshold_min \
-                    and avg_delta_ofp_graph < threshold_max\
-                    and int(i) > iter_ignore:
-                latency_ofp_to_graph_list.append(avg_delta_ofp_graph)
-            elif avg_delta_ofp_graph > (-10) and \
-                    avg_delta_ofp_graph < 0.0 and\
-                    int(i) > iter_ignore:
-                main.log.info("Sub-millisecond result likely; "+
-                    "negative result was rounded to 0")
-                #NOTE: Current metrics framework does not 
-                #support sub-millisecond accuracy. Therefore,
-                #if the result is negative, we can reasonably
-                #conclude sub-millisecond results and just 
-                #append the best rounded effort - 0 ms. 
-                latency_ofp_to_graph_list.append(0)
-            else:
-                main.log.info("Results for ofp-to-graph "+\
-                        "ignored due to excess in threshold")
-
-            #ofp to device processing latency (ONOS processing)
-            delta_ofp_device_1 = float(device_timestamp_1) - float(t0_ofp)
-            delta_ofp_device_2 = float(device_timestamp_2) - float(t0_ofp)
-            delta_ofp_device_3 = float(device_timestamp_3) - float(t0_ofp)
-            
-            avg_delta_ofp_device = \
-                    (float(delta_ofp_device_1)+\
-                     float(delta_ofp_device_2)+\
-                     float(delta_ofp_device_3)) / 3
-            
-            #NOTE: ofp - delta measurements are occasionally negative
-            #      due to system time misalignment.
-            latency_ofp_to_device_list.append(avg_delta_ofp_device)
-
-            delta_ofp_tcp = int(t0_ofp) - int(t0_tcp)
-            if delta_ofp_tcp > threshold_min \
-                    and delta_ofp_tcp < threshold_max and\
-                    int(i) > iter_ignore:
-                latency_tcp_to_ofp_list.append(delta_ofp_tcp)
-            else:
-                main.log.info("Results fo tcp-to-ofp "+\
-                        "ignored due to excess in threshold")
-
-            #TODO:
-            #Fetch logs upon threshold excess
-
-            main.log.info("ONOS1 delta end-to-end: "+
-                    str(delta_graph_1) + " ms")
-            main.log.info("ONOS2 delta end-to-end: "+
-                    str(delta_graph_2) + " ms")
-            main.log.info("ONOS3 delta end-to-end: "+
-                    str(delta_graph_3) + " ms")
-
-            main.log.info("ONOS1 delta OFP - graph: "+
-                    str(delta_ofp_graph_1) + " ms")
-            main.log.info("ONOS2 delta OFP - graph: "+
-                    str(delta_ofp_graph_2) + " ms")
-            main.log.info("ONOS3 delta OFP - graph: "+
-                    str(delta_ofp_graph_3) + " ms")
-            
-            main.log.info("ONOS1 delta device - t0: "+
-                    str(delta_device_1) + " ms")
-            main.log.info("ONOS2 delta device - t0: "+
-                    str(delta_device_2) + " ms")
-            main.log.info("ONOS3 delta device - t0: "+
-                    str(delta_device_3) + " ms")
-         
-            main.log.info("TCP to OFP delta: "+
-                    str(delta_ofp_tcp) + " ms")
-            #main.log.info("ONOS1 delta OFP - device: "+
-            #        str(delta_ofp_device_1) + " ms")
-            #main.log.info("ONOS2 delta OFP - device: "+
-            #        str(delta_ofp_device_2) + " ms")
-            #main.log.info("ONOS3 delta OFP - device: "+
-            #        str(delta_ofp_device_3) + " ms")
-
-            main.step("Remove switch from controller")
-            main.Mininet1.delete_sw_controller("s1")
-
-            time.sleep(5)
-
-        #END of for loop iteration
-
-        #If there is at least 1 element in each list,
-        #pass the test case
-        if len(latency_end_to_end_list) > 0 and\
-           len(latency_ofp_to_graph_list) > 0 and\
-           len(latency_ofp_to_device_list) > 0 and\
-           len(latency_t0_to_device_list) > 0 and\
-           len(latency_tcp_to_ofp_list) > 0:
-            assertion = main.TRUE
-        elif len(latency_end_to_end_list) == 0:
-            #The appending of 0 here is to prevent 
-            #the min,max,sum functions from failing 
-            #below
-            latency_end_to_end_list.append(0)
-            assertion = main.FALSE
-        elif len(latency_ofp_to_graph_list) == 0:
-            latency_ofp_to_graph_list.append(0)
-            assertion = main.FALSE
-        elif len(latency_ofp_to_device_list) == 0:
-            latency_ofp_to_device_list.append(0)
-            assertion = main.FALSE
-        elif len(latency_t0_to_device_list) == 0:
-            latency_t0_to_device_list.append(0)
-            assertion = main.FALSE
-        elif len(latency_tcp_to_ofp_list) == 0:
-            latency_tcp_to_ofp_list.append(0)
-            assertion = main.FALSE
-
-        #Calculate min, max, avg of latency lists
-        latency_end_to_end_max = \
-                int(max(latency_end_to_end_list))
-        latency_end_to_end_min = \
-                int(min(latency_end_to_end_list))
-        latency_end_to_end_avg = \
-                (int(sum(latency_end_to_end_list)) / \
-                 len(latency_end_to_end_list))
-        latency_end_to_end_std_dev = \
-                str(round(numpy.std(latency_end_to_end_list),1))
-
-        latency_ofp_to_graph_max = \
-                int(max(latency_ofp_to_graph_list))
-        latency_ofp_to_graph_min = \
-                int(min(latency_ofp_to_graph_list))
-        latency_ofp_to_graph_avg = \
-                (int(sum(latency_ofp_to_graph_list)) / \
-                 len(latency_ofp_to_graph_list))
-        latency_ofp_to_graph_std_dev = \
-                str(round(numpy.std(latency_ofp_to_graph_list),1))
-
-        latency_ofp_to_device_max = \
-                int(max(latency_ofp_to_device_list))
-        latency_ofp_to_device_min = \
-                int(min(latency_ofp_to_device_list))
-        latency_ofp_to_device_avg = \
-                (int(sum(latency_ofp_to_device_list)) / \
-                 len(latency_ofp_to_device_list))
-        latency_ofp_to_device_std_dev = \
-                str(round(numpy.std(latency_ofp_to_device_list),1))
-
-        latency_t0_to_device_max = \
-                int(max(latency_t0_to_device_list))
-        latency_t0_to_device_min = \
-                int(min(latency_t0_to_device_list))
-        latency_t0_to_device_avg = \
-                (int(sum(latency_t0_to_device_list)) / \
-                 len(latency_t0_to_device_list))
-        latency_ofp_to_device_std_dev = \
-                str(round(numpy.std(latency_t0_to_device_list),1))
-
-        latency_tcp_to_ofp_max = \
-                int(max(latency_tcp_to_ofp_list))
-        latency_tcp_to_ofp_min = \
-                int(min(latency_tcp_to_ofp_list))
-        latency_tcp_to_ofp_avg = \
-                (int(sum(latency_tcp_to_ofp_list)) / \
-                 len(latency_tcp_to_ofp_list))
-        latency_tcp_to_ofp_std_dev = \
-                str(round(numpy.std(latency_tcp_to_ofp_list),1))
-
-        main.log.report("Switch add - End-to-end latency: "+\
-                "Avg: "+str(latency_end_to_end_avg)+" ms "+
-                "Std Deviation: "+latency_end_to_end_std_dev+" ms")
-        main.log.report("Switch add - OFP-to-Graph latency: "+\
-                "Note: results are not accurate to sub-millisecond. "+
-                "Any sub-millisecond results are rounded to 0 ms. ")
-        main.log.report("Avg: "+str(latency_ofp_to_graph_avg)+" ms "+
-                "Std Deviation: "+latency_ofp_to_graph_std_dev+" ms")
-        main.log.report("Switch add - TCP-to-OFP latency: "+\
-                "Avg: "+str(latency_tcp_to_ofp_avg)+" ms "+
-                "Std Deviation: "+latency_tcp_to_ofp_std_dev+" ms")
-
-        if debug_mode == 'on':
-            main.ONOS1.cp_logs_to_dir("/opt/onos/log/karaf.log",
-                    "/tmp/", copy_file_name="sw_lat_karaf")
-
-        utilities.assert_equals(expect=main.TRUE, actual=assertion,
-                onpass="Switch latency test successful",
-                onfail="Switch latency test failed")
-
-    def CASE3(self, main):
-        '''
-        Bring port up / down and measure latency.
-        Port enable / disable is simulated by ifconfig up / down
-        
-        In ONOS-next, we must ensure that the port we are 
-        manipulating is connected to another switch with a valid
-        connection. Otherwise, graph view will not be updated.
-        '''
-        import time
-        import subprocess
-        import os
-        import requests
-        import json
-        import numpy
-
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS_user = main.params['CTRL']['user']
-
-        default_sw_port = main.params['CTRL']['port1']
-      
-        assertion = main.TRUE
-        #Number of iterations of case
-        num_iter = main.params['TEST']['numIter']
-       
-        #Timestamp 'keys' for json metrics output.
-        #These are subject to change, hence moved into params
-        deviceTimestamp = main.params['JSON']['deviceTimestamp']
-        graphTimestamp = main.params['JSON']['graphTimestamp']
-        
-        debug_mode = main.params['TEST']['debugMode']
-
-        local_time = time.strftime('%x %X')
-        local_time = local_time.replace("/","")
-        local_time = local_time.replace(" ","_")
-        local_time = local_time.replace(":","")
-        if debug_mode == 'on':
-            main.ONOS1.tshark_pcap("eth0",
-                    "/tmp/port_lat_pcap_"+local_time) 
-
-        #Threshold for this test case
-        up_threshold_str = main.params['TEST']['portUpThreshold']
-        down_threshold_str = main.params['TEST']['portDownThreshold']
-        
-        up_threshold_obj = up_threshold_str.split(",")
-        down_threshold_obj = down_threshold_str.split(",")
-
-        up_threshold_min = int(up_threshold_obj[0])
-        up_threshold_max = int(up_threshold_obj[1])
-
-        down_threshold_min = int(down_threshold_obj[0])
-        down_threshold_max = int(down_threshold_obj[1])
-
-        #NOTE: Some hardcoded variables you may need to configure
-        #      besides the params
-            
-        tshark_port_status = "OFP 130 Port Status"
-
-        tshark_port_up = "/tmp/tshark_port_up.txt"
-        tshark_port_down = "/tmp/tshark_port_down.txt"
-        interface_config = "s1-eth1"
-
-        main.log.report("Port enable / disable latency")
-        main.log.report("Simulated by ifconfig up / down")
-        main.log.report("Total iterations of test: "+str(num_iter))
-
-        main.step("Assign switches s1 and s2 to controller 1")
-        main.Mininet1.assign_sw_controller(sw="1",ip1=ONOS1_ip,
-                port1=default_sw_port)
-        main.Mininet1.assign_sw_controller(sw="2",ip1=ONOS1_ip,
-                port1=default_sw_port)
-
-        #Give enough time for metrics to propagate the 
-        #assign controller event. Otherwise, these events may
-        #carry over to our measurements
-        time.sleep(15)
-
-        port_up_device_to_ofp_list = []
-        port_up_graph_to_ofp_list = []
-        port_down_device_to_ofp_list = []
-        port_down_graph_to_ofp_list = []
-
-        for i in range(0, int(num_iter)):
-            main.step("Starting wireshark capture for port status down")
-            main.ONOS1.tshark_grep(tshark_port_status,
-                    tshark_port_down)
-            
-            time.sleep(5)
-
-            #Disable interface that is connected to switch 2
-            main.step("Disable port: "+interface_config)
-            main.Mininet1.handle.sendline("sh ifconfig "+
-                    interface_config+" down")
-            main.Mininet1.handle.expect("mininet>")
-
-            time.sleep(3)
-            main.ONOS1.tshark_stop()
-            
-            main.step("Obtain t1 by metrics call")
-            json_str_up_1 = main.ONOS1cli.topology_events_metrics()
-            json_str_up_2 = main.ONOS2cli.topology_events_metrics()
-            json_str_up_3 = main.ONOS3cli.topology_events_metrics()
-
-            json_obj_1 = json.loads(json_str_up_1)
-            json_obj_2 = json.loads(json_str_up_2)
-            json_obj_3 = json.loads(json_str_up_3)
-            
-            #Copy tshark output file from ONOS to TestON instance
-            #/tmp directory
-            os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
-                    tshark_port_down+" /tmp/")
-
-            f_port_down = open(tshark_port_down, 'r')
-            #Get first line of port down event from tshark
-            f_line = f_port_down.readline()
-            obj_down = f_line.split(" ")
-            if len(f_line) > 0:
-                timestamp_begin_pt_down = int(float(obj_down[1])*1000)
-                main.log.info("Port down begin timestamp: "+
-                        str(timestamp_begin_pt_down))
-            else:
-                main.log.info("Tshark output file returned unexpected"+
-                        " results: "+str(obj_down))
-                timestamp_begin_pt_down = 0
-            
-            f_port_down.close()
-
-            main.log.info("TEST tshark obj: "+str(obj_down))
-
-            time.sleep(3)
-
-            #Obtain graph timestamp. This timestsamp captures
-            #the epoch time at which the topology graph was updated.
-            graph_timestamp_1 = \
-                    json_obj_1[graphTimestamp]['value']
-            graph_timestamp_2 = \
-                    json_obj_2[graphTimestamp]['value']
-            graph_timestamp_3 = \
-                    json_obj_3[graphTimestamp]['value']
-
-            main.log.info("TEST graph timestamp ONOS1: "+
-                    str(graph_timestamp_1))
-
-            #Obtain device timestamp. This timestamp captures
-            #the epoch time at which the device event happened
-            device_timestamp_1 = \
-                    json_obj_1[deviceTimestamp]['value'] 
-            device_timestamp_2 = \
-                    json_obj_2[deviceTimestamp]['value'] 
-            device_timestamp_3 = \
-                    json_obj_3[deviceTimestamp]['value'] 
-
-            #Get delta between graph event and OFP 
-            pt_down_graph_to_ofp_1 = int(graph_timestamp_1) -\
-                    int(timestamp_begin_pt_down)
-            pt_down_graph_to_ofp_2 = int(graph_timestamp_2) -\
-                    int(timestamp_begin_pt_down)
-            pt_down_graph_to_ofp_3 = int(graph_timestamp_3) -\
-                    int(timestamp_begin_pt_down)
-
-            #Get delta between device event and OFP
-            pt_down_device_to_ofp_1 = int(device_timestamp_1) -\
-                    int(timestamp_begin_pt_down)
-            pt_down_device_to_ofp_2 = int(device_timestamp_2) -\
-                    int(timestamp_begin_pt_down)
-            pt_down_device_to_ofp_3 = int(device_timestamp_3) -\
-                    int(timestamp_begin_pt_down)
-       
-            #Caluclate average across clusters
-            pt_down_graph_to_ofp_avg =\
-                    (int(pt_down_graph_to_ofp_1) +
-                     int(pt_down_graph_to_ofp_2) + 
-                     int(pt_down_graph_to_ofp_3)) / 3
-            pt_down_device_to_ofp_avg = \
-                    (int(pt_down_device_to_ofp_1) + 
-                     int(pt_down_device_to_ofp_2) +
-                     int(pt_down_device_to_ofp_3)) / 3
-
-            if pt_down_graph_to_ofp_avg > down_threshold_min and \
-                    pt_down_graph_to_ofp_avg < down_threshold_max:
-                port_down_graph_to_ofp_list.append(
-                    pt_down_graph_to_ofp_avg)
-                main.log.info("Port down: graph to ofp avg: "+
-                    str(pt_down_graph_to_ofp_avg) + " ms")
-            else:
-                main.log.info("Average port down graph-to-ofp result" +
-                        " exceeded the threshold: "+
-                        str(pt_down_graph_to_ofp_avg))
-
-            if pt_down_device_to_ofp_avg > 0 and \
-                    pt_down_device_to_ofp_avg < 1000:
-                port_down_device_to_ofp_list.append(
-                    pt_down_device_to_ofp_avg)
-                main.log.info("Port down: device to ofp avg: "+
-                    str(pt_down_device_to_ofp_avg) + " ms")
-            else:
-                main.log.info("Average port down device-to-ofp result" +
-                        " exceeded the threshold: "+
-                        str(pt_down_device_to_ofp_avg))
-
-            #Port up events 
-            main.step("Enable port and obtain timestamp")
-            main.step("Starting wireshark capture for port status up")
-            main.ONOS1.tshark_grep(tshark_port_status, tshark_port_up)
-            time.sleep(5)
-
-            main.Mininet1.handle.sendline("sh ifconfig "+
-                    interface_config+" up")
-            main.Mininet1.handle.expect("mininet>")
-            
-            #Allow time for tshark to capture event
-            time.sleep(3)
-            main.ONOS1.tshark_stop()
-
-            #Obtain metrics shortly afterwards
-            #This timestsamp captures
-            #the epoch time at which the topology graph was updated.
-            main.step("Obtain t1 by REST call")
-            json_str_up_1 = main.ONOS1cli.topology_events_metrics()
-            json_str_up_2 = main.ONOS2cli.topology_events_metrics()
-            json_str_up_3 = main.ONOS3cli.topology_events_metrics()
-            
-            json_obj_1 = json.loads(json_str_up_1)
-            json_obj_2 = json.loads(json_str_up_2)
-            json_obj_3 = json.loads(json_str_up_3)
-
-            os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
-                    tshark_port_up+" /tmp/")
-
-            f_port_up = open(tshark_port_up, 'r')
-            f_line = f_port_up.readline()
-            obj_up = f_line.split(" ")
-            if len(f_line) > 0:
-                timestamp_begin_pt_up = int(float(obj_up[1])*1000)
-                main.log.info("Port up begin timestamp: "+
-                        str(timestamp_begin_pt_up))
-            else:
-                main.log.info("Tshark output file returned unexpected"+
-                        " results.")
-                timestamp_begin_pt_up = 0
-            
-            f_port_up.close()
-
-            graph_timestamp_1 = \
-                    json_obj_1[graphTimestamp]['value']
-            graph_timestamp_2 = \
-                    json_obj_2[graphTimestamp]['value']
-            graph_timestamp_3 = \
-                    json_obj_3[graphTimestamp]['value']
-
-            #Obtain device timestamp. This timestamp captures
-            #the epoch time at which the device event happened
-            device_timestamp_1 = \
-                    json_obj_1[deviceTimestamp]['value'] 
-            device_timestamp_2 = \
-                    json_obj_2[deviceTimestamp]['value'] 
-            device_timestamp_3 = \
-                    json_obj_3[deviceTimestamp]['value'] 
-
-            #Get delta between graph event and OFP 
-            pt_up_graph_to_ofp_1 = int(graph_timestamp_1) -\
-                    int(timestamp_begin_pt_up)
-            pt_up_graph_to_ofp_2 = int(graph_timestamp_2) -\
-                    int(timestamp_begin_pt_up)
-            pt_up_graph_to_ofp_3 = int(graph_timestamp_3) -\
-                    int(timestamp_begin_pt_up)
-
-            #Get delta between device event and OFP
-            pt_up_device_to_ofp_1 = int(device_timestamp_1) -\
-                    int(timestamp_begin_pt_up)
-            pt_up_device_to_ofp_2 = int(device_timestamp_2) -\
-                    int(timestamp_begin_pt_up)
-            pt_up_device_to_ofp_3 = int(device_timestamp_3) -\
-                    int(timestamp_begin_pt_up)
-
-            main.log.info("ONOS1 delta G2O: "+str(pt_up_graph_to_ofp_1))
-            main.log.info("ONOS2 delta G2O: "+str(pt_up_graph_to_ofp_2))
-            main.log.info("ONOS3 delta G2O: "+str(pt_up_graph_to_ofp_3))
-
-            main.log.info("ONOS1 delta D2O: "+str(pt_up_device_to_ofp_1))
-            main.log.info("ONOS2 delta D2O: "+str(pt_up_device_to_ofp_2)) 
-            main.log.info("ONOS3 delta D2O: "+str(pt_up_device_to_ofp_3)) 
-
-            pt_up_graph_to_ofp_avg = \
-                    (int(pt_up_graph_to_ofp_1) + 
-                     int(pt_up_graph_to_ofp_2) +
-                     int(pt_up_graph_to_ofp_3)) / 3
-
-            pt_up_device_to_ofp_avg = \
-                    (int(pt_up_device_to_ofp_1) + 
-                     int(pt_up_device_to_ofp_2) +
-                     int(pt_up_device_to_ofp_3)) / 3
-
-            if pt_up_graph_to_ofp_avg > up_threshold_min and \
-                    pt_up_graph_to_ofp_avg < up_threshold_max: 
-                port_up_graph_to_ofp_list.append(
-                        pt_up_graph_to_ofp_avg)
-                main.log.info("Port down: graph to ofp avg: "+
-                    str(pt_up_graph_to_ofp_avg) + " ms")
-            else:
-                main.log.info("Average port up graph-to-ofp result"+
-                        " exceeded the threshold: "+
-                        str(pt_up_graph_to_ofp_avg))
-            
-            if pt_up_device_to_ofp_avg > up_threshold_min and \
-                    pt_up_device_to_ofp_avg < up_threshold_max:
-                port_up_device_to_ofp_list.append(
-                        pt_up_device_to_ofp_avg)
-                main.log.info("Port up: device to ofp avg: "+
-                    str(pt_up_device_to_ofp_avg) + " ms")
-            else:
-                main.log.info("Average port up device-to-ofp result"+
-                        " exceeded the threshold: "+
-                        str(pt_up_device_to_ofp_avg))
-            
-            #END ITERATION FOR LOOP
-        
-        #Check all list for latency existence and set assertion
-        if (port_down_graph_to_ofp_list and port_down_device_to_ofp_list\
-           and port_up_graph_to_ofp_list and port_up_device_to_ofp_list):
-            assertion = main.TRUE
-
-        #Calculate and report latency measurements
-        port_down_graph_to_ofp_min = min(port_down_graph_to_ofp_list)
-        port_down_graph_to_ofp_max = max(port_down_graph_to_ofp_list)
-        port_down_graph_to_ofp_avg = \
-                (sum(port_down_graph_to_ofp_list) / 
-                 len(port_down_graph_to_ofp_list))
-        port_down_graph_to_ofp_std_dev = \
-                str(round(numpy.std(port_down_graph_to_ofp_list),1))
-        
-        main.log.report("Port down graph-to-ofp "+
-                "Avg: "+str(port_down_graph_to_ofp_avg)+" ms "+
-                "Std Deviation: "+port_down_graph_to_ofp_std_dev+" ms")
-        
-        port_down_device_to_ofp_min = min(port_down_device_to_ofp_list)
-        port_down_device_to_ofp_max = max(port_down_device_to_ofp_list)
-        port_down_device_to_ofp_avg = \
-                (sum(port_down_device_to_ofp_list) /\
-                 len(port_down_device_to_ofp_list))
-        port_down_device_to_ofp_std_dev = \
-                str(round(numpy.std(port_down_device_to_ofp_list),1))
-        
-        main.log.report("Port down device-to-ofp "+
-                "Avg: "+str(port_down_device_to_ofp_avg)+" ms "+
-                "Std Deviation: "+port_down_device_to_ofp_std_dev+" ms")
-        
-        port_up_graph_to_ofp_min = min(port_up_graph_to_ofp_list)
-        port_up_graph_to_ofp_max = max(port_up_graph_to_ofp_list)
-        port_up_graph_to_ofp_avg = \
-                (sum(port_up_graph_to_ofp_list) /\
-                 len(port_up_graph_to_ofp_list))
-        port_up_graph_to_ofp_std_dev = \
-                str(round(numpy.std(port_up_graph_to_ofp_list),1))
-        
-        main.log.report("Port up graph-to-ofp "+
-                "Avg: "+str(port_up_graph_to_ofp_avg)+" ms "+
-                "Std Deviation: "+port_up_graph_to_ofp_std_dev+" ms")
-          
-        port_up_device_to_ofp_min = min(port_up_device_to_ofp_list)
-        port_up_device_to_ofp_max = max(port_up_device_to_ofp_list)
-        port_up_device_to_ofp_avg = \
-                (sum(port_up_device_to_ofp_list) /\
-                 len(port_up_device_to_ofp_list))
-        port_up_device_to_ofp_std_dev = \
-                str(round(numpy.std(port_up_device_to_ofp_list),1))
-        
-        main.log.report("Port up device-to-ofp "+
-                "Avg: "+str(port_up_device_to_ofp_avg)+" ms "+
-                "Std Deviation: "+port_up_device_to_ofp_std_dev+" ms")
-
-        utilities.assert_equals(expect=main.TRUE, actual=assertion,
-                onpass="Port discovery latency calculation successful",
-                onfail="Port discovery test failed")
-
-    def CASE4(self, main):
-        '''
-        Link down event using loss rate 100%
-        
-        Important:
-            Use a simple 2 switch topology with 1 link between
-            the two switches. Ensure that mac addresses of the 
-            switches are 1 / 2 respectively
-        '''
-        import time
-        import subprocess
-        import os
-        import requests
-        import json
-        import numpy 
-    
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS_user = main.params['CTRL']['user']
-
-        default_sw_port = main.params['CTRL']['port1']
-       
-        #Number of iterations of case
-        num_iter = main.params['TEST']['numIter']
-       
-        #Timestamp 'keys' for json metrics output.
-        #These are subject to change, hence moved into params
-        deviceTimestamp = main.params['JSON']['deviceTimestamp']
-        linkTimestamp = main.params['JSON']['linkTimestamp'] 
-        graphTimestamp = main.params['JSON']['graphTimestamp']
-        
-        debug_mode = main.params['TEST']['debugMode']
-
-        local_time = time.strftime('%x %X')
-        local_time = local_time.replace("/","")
-        local_time = local_time.replace(" ","_")
-        local_time = local_time.replace(":","")
-        if debug_mode == 'on':
-            main.ONOS1.tshark_pcap("eth0",
-                    "/tmp/link_lat_pcap_"+local_time) 
-
-        #Threshold for this test case
-        up_threshold_str = main.params['TEST']['linkUpThreshold']
-        down_threshold_str = main.params['TEST']['linkDownThreshold']
-
-        up_threshold_obj = up_threshold_str.split(",")
-        down_threshold_obj = down_threshold_str.split(",")
-
-        up_threshold_min = int(up_threshold_obj[0])
-        up_threshold_max = int(up_threshold_obj[1])
-
-        down_threshold_min = int(down_threshold_obj[0])
-        down_threshold_max = int(down_threshold_obj[1])
-
-        assertion = main.TRUE
-        #Link event timestamp to system time list
-        link_down_link_to_system_list = []
-        link_up_link_to_system_list = []
-        #Graph event timestamp to system time list
-        link_down_graph_to_system_list = []
-        link_up_graph_to_system_list = [] 
-
-        main.log.report("Link up / down discovery latency between "+
-                "two switches")
-        main.log.report("Simulated by setting loss-rate 100%")
-        main.log.report("'tc qdisc add dev <intfs> root netem loss 100%'") 
-        main.log.report("Total iterations of test: "+str(num_iter))
-
-        main.step("Assign all switches")
-        main.Mininet1.assign_sw_controller(sw="1",
-                ip1=ONOS1_ip, port1=default_sw_port)
-        main.Mininet1.assign_sw_controller(sw="2",
-                ip1=ONOS1_ip, port1=default_sw_port)
-
-        main.step("Verifying switch assignment")
-        result_s1 = main.Mininet1.get_sw_controller(sw="s1")
-        result_s2 = main.Mininet1.get_sw_controller(sw="s2")
-          
-        #Allow time for events to finish before taking measurements
-        time.sleep(10)
-
-        link_down1 = False
-        link_down2 = False
-        link_down3 = False
-        #Start iteration of link event test
-        for i in range(0, int(num_iter)):
-            main.step("Getting initial system time as t0")
-        
-            #System time in epoch ms
-            timestamp_link_down_t0 = time.time() * 1000
-            #Link down is simulated by 100% loss rate using traffic 
-            #control command
-            main.Mininet1.handle.sendline(
-                    "sh tc qdisc add dev s1-eth1 root netem loss 100%")
-
-            #TODO: Iterate through 'links' command to verify that
-            #      link s1 -> s2 went down (loop timeout 30 seconds) 
-            #      on all 3 ONOS instances
-            main.log.info("Checking ONOS for link update")
-            loop_count = 0
-            while( not (link_down1 and link_down2 and link_down3)\
-                    and loop_count < 30 ):
-                json_str1 = main.ONOS1cli.links()
-                json_str2 = main.ONOS2cli.links()
-                json_str3 = main.ONOS3cli.links()
-                
-                if not (json_str1 and json_str2 and json_str3):
-                    main.log.error("CLI command returned error ")
-                    break
-                else:
-                    json_obj1 = json.loads(json_str1)
-                    json_obj2 = json.loads(json_str2)
-                    json_obj3 = json.loads(json_str3)
-                for obj1 in json_obj1:
-                    if '01' not in obj1['src']['device']:
-                        link_down1 = True
-                        main.log.info("Link down from "+
-                                "s1 -> s2 on ONOS1 detected")
-                for obj2 in json_obj2:
-                    if '01' not in obj2['src']['device']:
-                        link_down2 = True
-                        main.log.info("Link down from "+
-                                "s1 -> s2 on ONOS2 detected")
-                for obj3 in json_obj3:
-                    if '01' not in obj3['src']['device']:
-                        link_down3 = True
-                        main.log.info("Link down from "+
-                                "s1 -> s2 on ONOS3 detected")
-                
-                loop_count += 1
-                #If CLI doesn't like the continuous requests
-                #and exits in this loop, increase the sleep here.
-                #Consequently, while loop timeout will increase
-                time.sleep(1)
-    
-            #Give time for metrics measurement to catch up
-            #NOTE: May need to be configured more accurately
-            time.sleep(10)
-            #If we exited the while loop and link down 1,2,3 are still 
-            #false, then ONOS has failed to discover link down event
-            if not (link_down1 and link_down2 and link_down3):
-                main.log.info("Link down discovery failed")
-                
-                link_down_lat_graph1 = 0
-                link_down_lat_graph2 = 0
-                link_down_lat_graph3 = 0
-                link_down_lat_device1 = 0
-                link_down_lat_device2 = 0
-                link_down_lat_device3 = 0
-                
-                assertion = main.FALSE
-            else:
-                json_topo_metrics_1 =\
-                        main.ONOS1cli.topology_events_metrics()
-                json_topo_metrics_2 =\
-                        main.ONOS2cli.topology_events_metrics()
-                json_topo_metrics_3 =\
-                        main.ONOS3cli.topology_events_metrics()
-                json_topo_metrics_1 = json.loads(json_topo_metrics_1)
-                json_topo_metrics_2 = json.loads(json_topo_metrics_2)
-                json_topo_metrics_3 = json.loads(json_topo_metrics_3)
-
-                main.log.info("Obtaining graph and device timestamp")
-                graph_timestamp_1 = \
-                    json_topo_metrics_1[graphTimestamp]['value']
-                graph_timestamp_2 = \
-                    json_topo_metrics_2[graphTimestamp]['value']
-                graph_timestamp_3 = \
-                    json_topo_metrics_3[graphTimestamp]['value']
-
-                link_timestamp_1 = \
-                    json_topo_metrics_1[linkTimestamp]['value']
-                link_timestamp_2 = \
-                    json_topo_metrics_2[linkTimestamp]['value']
-                link_timestamp_3 = \
-                    json_topo_metrics_3[linkTimestamp]['value']
-
-                if graph_timestamp_1 and graph_timestamp_2 and\
-                        graph_timestamp_3 and link_timestamp_1 and\
-                        link_timestamp_2 and link_timestamp_3:
-                    link_down_lat_graph1 = int(graph_timestamp_1) -\
-                            int(timestamp_link_down_t0)
-                    link_down_lat_graph2 = int(graph_timestamp_2) -\
-                            int(timestamp_link_down_t0)
-                    link_down_lat_graph3 = int(graph_timestamp_3) -\
-                            int(timestamp_link_down_t0)
-                
-                    link_down_lat_link1 = int(link_timestamp_1) -\
-                            int(timestamp_link_down_t0)
-                    link_down_lat_link2 = int(link_timestamp_2) -\
-                            int(timestamp_link_down_t0)
-                    link_down_lat_link3 = int(link_timestamp_3) -\
-                            int(timestamp_link_down_t0)
-                else:
-                    main.log.error("There was an error calculating"+
-                        " the delta for link down event")
-                    link_down_lat_graph1 = 0
-                    link_down_lat_graph2 = 0
-                    link_down_lat_graph3 = 0
-                    
-                    link_down_lat_device1 = 0
-                    link_down_lat_device2 = 0
-                    link_down_lat_device3 = 0
-        
-            main.log.info("Link down latency ONOS1 iteration "+
-                    str(i)+" (end-to-end): "+
-                    str(link_down_lat_graph1)+" ms")
-            main.log.info("Link down latency ONOS2 iteration "+
-                    str(i)+" (end-to-end): "+
-                    str(link_down_lat_graph2)+" ms")
-            main.log.info("Link down latency ONOS3 iteration "+
-                    str(i)+" (end-to-end): "+
-                    str(link_down_lat_graph3)+" ms")
-            
-            main.log.info("Link down latency ONOS1 iteration "+
-                    str(i)+" (link-event-to-system-timestamp): "+
-                    str(link_down_lat_link1)+" ms")
-            main.log.info("Link down latency ONOS2 iteration "+
-                    str(i)+" (link-event-to-system-timestamp): "+
-                    str(link_down_lat_link2)+" ms")
-            main.log.info("Link down latency ONOS3 iteration "+
-                    str(i)+" (link-event-to-system-timestamp): "+
-                    str(link_down_lat_link3))
-      
-            #Calculate avg of node calculations
-            link_down_lat_graph_avg =\
-                    (link_down_lat_graph1 +
-                     link_down_lat_graph2 +
-                     link_down_lat_graph3) / 3
-            link_down_lat_link_avg =\
-                    (link_down_lat_link1 +
-                     link_down_lat_link2 +
-                     link_down_lat_link3) / 3
-
-            #Set threshold and append latency to list
-            if link_down_lat_graph_avg > down_threshold_min and\
-               link_down_lat_graph_avg < down_threshold_max:
-                link_down_graph_to_system_list.append(
-                        link_down_lat_graph_avg)
-            else:
-                main.log.info("Link down latency exceeded threshold")
-                main.log.info("Results for iteration "+str(i)+
-                        "have been omitted")
-            if link_down_lat_link_avg > down_threshold_min and\
-               link_down_lat_link_avg < down_threshold_max:
-                link_down_link_to_system_list.append(
-                        link_down_lat_link_avg)
-            else:
-                main.log.info("Link down latency exceeded threshold")
-                main.log.info("Results for iteration "+str(i)+
-                        "have been omitted")
-
-            #NOTE: To remove loss rate and measure latency:
-            #       'sh tc qdisc del dev s1-eth1 root'
-            timestamp_link_up_t0 = time.time() * 1000
-            main.Mininet1.handle.sendline("sh tc qdisc del dev "+
-                    "s1-eth1 root")
-            main.Mininet1.handle.expect("mininet>")
-            
-            main.log.info("Checking ONOS for link update")
-            
-            link_down1 = True
-            link_down2 = True
-            link_down3 = True
-            loop_count = 0
-            while( (link_down1 and link_down2 and link_down3)\
-                    and loop_count < 30 ):
-                json_str1 = main.ONOS1cli.links()
-                json_str2 = main.ONOS2cli.links()
-                json_str3 = main.ONOS3cli.links()
-                if not (json_str1 and json_str2 and json_str3):
-                    main.log.error("CLI command returned error ")
-                    break
-                else:
-                    json_obj1 = json.loads(json_str1)
-                    json_obj2 = json.loads(json_str2)
-                    json_obj3 = json.loads(json_str3)
-                
-                for obj1 in json_obj1:
-                    if '01' in obj1['src']['device']:
-                        link_down1 = False 
-                        main.log.info("Link up from "+
-                            "s1 -> s2 on ONOS1 detected")
-                for obj2 in json_obj2:
-                    if '01' in obj2['src']['device']:
-                        link_down2 = False 
-                        main.log.info("Link up from "+
-                            "s1 -> s2 on ONOS2 detected")
-                for obj3 in json_obj3:
-                    if '01' in obj3['src']['device']:
-                        link_down3 = False 
-                        main.log.info("Link up from "+
-                            "s1 -> s2 on ONOS3 detected")
-                
-                loop_count += 1
-                time.sleep(1)
-            
-            if (link_down1 and link_down2 and link_down3):
-                main.log.info("Link up discovery failed")
-                
-                link_up_lat_graph1 = 0
-                link_up_lat_graph2 = 0
-                link_up_lat_graph3 = 0
-                link_up_lat_device1 = 0
-                link_up_lat_device2 = 0
-                link_up_lat_device3 = 0
-                
-                assertion = main.FALSE
-            else:
-                json_topo_metrics_1 =\
-                        main.ONOS1cli.topology_events_metrics()
-                json_topo_metrics_2 =\
-                        main.ONOS2cli.topology_events_metrics()
-                json_topo_metrics_3 =\
-                        main.ONOS3cli.topology_events_metrics()
-                json_topo_metrics_1 = json.loads(json_topo_metrics_1)
-                json_topo_metrics_2 = json.loads(json_topo_metrics_2)
-                json_topo_metrics_3 = json.loads(json_topo_metrics_3)
-
-                main.log.info("Obtaining graph and device timestamp")
-                graph_timestamp_1 = \
-                    json_topo_metrics_1[graphTimestamp]['value']
-                graph_timestamp_2 = \
-                    json_topo_metrics_2[graphTimestamp]['value']
-                graph_timestamp_3 = \
-                    json_topo_metrics_3[graphTimestamp]['value']
-
-                link_timestamp_1 = \
-                    json_topo_metrics_1[linkTimestamp]['value']
-                link_timestamp_2 = \
-                    json_topo_metrics_2[linkTimestamp]['value']
-                link_timestamp_3 = \
-                    json_topo_metrics_3[linkTimestamp]['value']
-
-                if graph_timestamp_1 and graph_timestamp_2 and\
-                        graph_timestamp_3 and link_timestamp_1 and\
-                        link_timestamp_2 and link_timestamp_3:
-                    link_up_lat_graph1 = int(graph_timestamp_1) -\
-                            int(timestamp_link_up_t0)
-                    link_up_lat_graph2 = int(graph_timestamp_2) -\
-                            int(timestamp_link_up_t0)
-                    link_up_lat_graph3 = int(graph_timestamp_3) -\
-                            int(timestamp_link_up_t0)
-                
-                    link_up_lat_link1 = int(link_timestamp_1) -\
-                            int(timestamp_link_up_t0)
-                    link_up_lat_link2 = int(link_timestamp_2) -\
-                            int(timestamp_link_up_t0)
-                    link_up_lat_link3 = int(link_timestamp_3) -\
-                            int(timestamp_link_up_t0)
-                else:
-                    main.log.error("There was an error calculating"+
-                        " the delta for link down event")
-                    link_up_lat_graph1 = 0
-                    link_up_lat_graph2 = 0
-                    link_up_lat_graph3 = 0
-                    
-                    link_up_lat_device1 = 0
-                    link_up_lat_device2 = 0
-                    link_up_lat_device3 = 0
-       
-            if debug_mode == 'on':
-                main.log.info("Link up latency ONOS1 iteration "+
-                    str(i)+" (end-to-end): "+
-                    str(link_up_lat_graph1)+" ms")
-                main.log.info("Link up latency ONOS2 iteration "+
-                    str(i)+" (end-to-end): "+
-                    str(link_up_lat_graph2)+" ms")
-                main.log.info("Link up latency ONOS3 iteration "+
-                    str(i)+" (end-to-end): "+
-                    str(link_up_lat_graph3)+" ms")
-            
-                main.log.info("Link up latency ONOS1 iteration "+
-                    str(i)+" (link-event-to-system-timestamp): "+
-                    str(link_up_lat_link1)+" ms")
-                main.log.info("Link up latency ONOS2 iteration "+
-                    str(i)+" (link-event-to-system-timestamp): "+
-                    str(link_up_lat_link2)+" ms")
-                main.log.info("Link up latency ONOS3 iteration "+
-                    str(i)+" (link-event-to-system-timestamp): "+
-                    str(link_up_lat_link3))
-      
-            #Calculate avg of node calculations
-            link_up_lat_graph_avg =\
-                    (link_up_lat_graph1 +
-                     link_up_lat_graph2 +
-                     link_up_lat_graph3) / 3
-            link_up_lat_link_avg =\
-                    (link_up_lat_link1 +
-                     link_up_lat_link2 +
-                     link_up_lat_link3) / 3
-
-            #Set threshold and append latency to list
-            if link_up_lat_graph_avg > up_threshold_min and\
-               link_up_lat_graph_avg < up_threshold_max:
-                link_up_graph_to_system_list.append(
-                        link_up_lat_graph_avg)
-            else:
-                main.log.info("Link up latency exceeded threshold")
-                main.log.info("Results for iteration "+str(i)+
-                        "have been omitted")
-            if link_up_lat_link_avg > up_threshold_min and\
-               link_up_lat_link_avg < up_threshold_max:
-                link_up_link_to_system_list.append(
-                        link_up_lat_link_avg)
-            else:
-                main.log.info("Link up latency exceeded threshold")
-                main.log.info("Results for iteration "+str(i)+
-                        "have been omitted")
-
-        #Calculate min, max, avg of list and report
-        link_down_min = min(link_down_graph_to_system_list)
-        link_down_max = max(link_down_graph_to_system_list)
-        link_down_avg = sum(link_down_graph_to_system_list) / \
-                        len(link_down_graph_to_system_list)
-        link_up_min = min(link_up_graph_to_system_list)
-        link_up_max = max(link_up_graph_to_system_list)
-        link_up_avg = sum(link_up_graph_to_system_list) / \
-                        len(link_up_graph_to_system_list)
-        link_down_std_dev = \
-                str(round(numpy.std(link_down_graph_to_system_list),1))
-        link_up_std_dev = \
-                str(round(numpy.std(link_up_graph_to_system_list),1))
-
-        main.log.report("Link down latency " +
-                "Avg: "+str(link_down_avg)+" ms "+
-                "Std Deviation: "+link_down_std_dev+" ms")
-        main.log.report("Link up latency "+
-                "Avg: "+str(link_up_avg)+" ms "+
-                "Std Deviation: "+link_up_std_dev+" ms")
-
-        utilities.assert_equals(expect=main.TRUE, actual=assertion,
-                onpass="Link discovery latency calculation successful",
-                onfail="Link discovery latency case failed")
-
-    def CASE5(self, main):
-        '''
-        100 Switch discovery latency
-
-        Important:
-            This test case can be potentially dangerous if 
-            your machine has previously set iptables rules.
-            One of the steps of the test case will flush
-            all existing iptables rules.
-        Note:
-            You can specify the number of switches in the 
-            params file to adjust the switch discovery size
-            (and specify the corresponding topology in Mininet1 
-            .topo file)
-        '''
-        import time
-        import subprocess
-        import os
-        import requests
-        import json
-
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        MN1_ip = main.params['MN']['ip1']
-        ONOS_user = main.params['CTRL']['user']
-
-        default_sw_port = main.params['CTRL']['port1']
-       
-        #Number of iterations of case
-        num_iter = main.params['TEST']['numIter']
-        num_sw = main.params['TEST']['numSwitch']
-
-        #Timestamp 'keys' for json metrics output.
-        #These are subject to change, hence moved into params
-        deviceTimestamp = main.params['JSON']['deviceTimestamp']
-        graphTimestamp = main.params['JSON']['graphTimestamp']
-        
-        debug_mode = main.params['TEST']['debugMode']
-
-        local_time = time.strftime('%X')
-        local_time = local_time.replace("/","")
-        local_time = local_time.replace(" ","_")
-        local_time = local_time.replace(":","")
-        if debug_mode == 'on':
-            main.ONOS1.tshark_pcap("eth0",
-                    "/tmp/100_sw_lat_pcap_"+local_time) 
- 
-        #Threshold for this test case
-        sw_disc_threshold_str = main.params['TEST']['swDisc100Threshold']
-        sw_disc_threshold_obj = sw_disc_threshold_str.split(",")
-        sw_disc_threshold_min = int(sw_disc_threshold_obj[0])
-        sw_disc_threshold_max = int(sw_disc_threshold_obj[1])
-
-        tshark_ofp_output = "/tmp/tshark_ofp_"+num_sw+"sw.txt"
-        tshark_tcp_output = "/tmp/tshark_tcp_"+num_sw+"sw.txt"
-
-        tshark_ofp_result_list = []
-        tshark_tcp_result_list = []
-
-        sw_discovery_lat_list = []
-
-        main.case(num_sw+" Switch discovery latency")
-        main.step("Assigning all switches to ONOS1")
-        for i in range(1, int(num_sw)+1):
-            main.Mininet1.assign_sw_controller(
-                    sw=str(i),
-                    ip1=ONOS1_ip,
-                    port1=default_sw_port)
-        
-        #Ensure that nodes are configured with ptpd
-        #Just a warning message
-        main.log.info("Please check ptpd configuration to ensure"+\
-                " All nodes' system times are in sync")
-        time.sleep(5)
-
-        for i in range(0, int(num_iter)):
-            
-            main.step("Set iptables rule to block incoming sw connections")
-            #Set iptables rule to block incoming switch connections
-            #The rule description is as follows:
-            #   Append to INPUT rule,
-            #   behavior DROP that matches following:
-            #       * packet type: tcp
-            #       * source IP: MN1_ip
-            #       * destination PORT: 6633
-            main.ONOS1.handle.sendline(
-                    "sudo iptables -A INPUT -p tcp -s "+MN1_ip+
-                    " --dport "+default_sw_port+" -j DROP")
-            main.ONOS1.handle.expect("\$") 
-            #   Append to OUTPUT rule, 
-            #   behavior DROP that matches following:
-            #       * packet type: tcp
-            #       * source IP: MN1_ip
-            #       * destination PORT: 6633
-            main.ONOS1.handle.sendline(
-                    "sudo iptables -A OUTPUT -p tcp -s "+MN1_ip+
-                    " --dport "+default_sw_port+" -j DROP")
-            main.ONOS1.handle.expect("\$")
-            #Give time to allow rule to take effect
-            #NOTE: Sleep period may need to be configured 
-            #      based on the number of switches in the topology
-            main.log.info("Please wait for switch connection to "+
-                    "time out")
-            time.sleep(60)
-            
-            #Gather vendor OFP with tshark
-            main.ONOS1.tshark_grep("OFP 86 Vendor", 
-                    tshark_ofp_output)
-            main.ONOS1.tshark_grep("TCP 74 ",
-                    tshark_tcp_output)
-
-            #NOTE: Remove all iptables rule quickly (flush)
-            #      Before removal, obtain TestON timestamp at which 
-            #      removal took place
-            #      (ensuring nodes are configured via ptp)
-            #      sudo iptables -F
-            
-            t0_system = time.time() * 1000
-            main.ONOS1.handle.sendline(
-                    "sudo iptables -F")
-
-            #Counter to track loop count
-            counter_loop = 0
-            counter_avail1 = 0
-            counter_avail2 = 0
-            counter_avail3 = 0
-            onos1_dev = False
-            onos2_dev = False
-            onos3_dev = False
-            while counter_loop < 60:
-                #Continue to check devices for all device 
-                #availability. When all devices in all 3
-                #ONOS instances indicate that devices are available
-                #obtain graph event timestamp for t1.
-                device_str_obj1 = main.ONOS1cli.devices()
-                device_str_obj2 = main.ONOS2cli.devices()
-                device_str_obj3 = main.ONOS3cli.devices()
-
-                device_json1 = json.loads(device_str_obj1)                
-                device_json2 = json.loads(device_str_obj2)                
-                device_json3 = json.loads(device_str_obj3)           
-                
-                for device1 in device_json1:
-                    if device1['available'] == True:
-                        counter_avail1 += 1
-                        if counter_avail1 == int(num_sw):
-                            onos1_dev = True
-                            main.log.info("All devices have been "+
-                                    "discovered on ONOS1")
-                    else:
-                        counter_avail1 = 0
-                for device2 in device_json2:
-                    if device2['available'] == True:
-                        counter_avail2 += 1
-                        if counter_avail2 == int(num_sw):
-                            onos2_dev = True
-                            main.log.info("All devices have been "+
-                                    "discovered on ONOS2")
-                    else:
-                        counter_avail2 = 0
-                for device3 in device_json3:
-                    if device3['available'] == True:
-                        counter_avail3 += 1
-                        if counter_avail3 == int(num_sw):
-                            onos3_dev = True
-                            main.log.info("All devices have been "+
-                                    "discovered on ONOS3")
-                    else:
-                        counter_avail3 = 0
-
-                if onos1_dev and onos2_dev and onos3_dev:
-                    main.log.info("All devices have been discovered "+
-                            "on all ONOS instances")
-                    json_str_topology_metrics_1 =\
-                        main.ONOS1cli.topology_events_metrics()
-                    json_str_topology_metrics_2 =\
-                        main.ONOS2cli.topology_events_metrics()
-                    json_str_topology_metrics_3 =\
-                        main.ONOS3cli.topology_events_metrics()
-                   
-                    #Exit while loop if all devices discovered
-                    break 
-                
-                counter_loop += 1
-                #Give some time in between CLI calls
-                #(will not affect measurement)
-                time.sleep(3)
-
-            main.ONOS1.tshark_stop()
-            
-            os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
-                    tshark_ofp_output+" /tmp/") 
-            os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
-                    tshark_tcp_output+" /tmp/")
-
-            #TODO: Automate OFP output analysis
-            #Debug mode - print out packets captured at runtime     
-            if debug_mode == 'on': 
-                ofp_file = open(tshark_ofp_output, 'r')
-                main.log.info("Tshark OFP Vendor output: ")
-                for line in ofp_file:
-                    tshark_ofp_result_list.append(line)
-                    main.log.info(line)
-                ofp_file.close()
-
-                tcp_file = open(tshark_tcp_output, 'r')
-                main.log.info("Tshark TCP 74 output: ")
-                for line in tcp_file:
-                    tshark_tcp_result_list.append(line)
-                    main.log.info(line)
-                tcp_file.close()
-
-            json_obj_1 = json.loads(json_str_topology_metrics_1)
-            json_obj_2 = json.loads(json_str_topology_metrics_2)
-            json_obj_3 = json.loads(json_str_topology_metrics_3)
-
-            graph_timestamp_1 = \
-                    json_obj_1[graphTimestamp]['value']
-            graph_timestamp_2 = \
-                    json_obj_2[graphTimestamp]['value']
-            graph_timestamp_3 = \
-                    json_obj_3[graphTimestamp]['value']
-
-            graph_lat_1 = int(graph_timestamp_1) - int(t0_system)
-            graph_lat_2 = int(graph_timestamp_2) - int(t0_system)
-            graph_lat_3 = int(graph_timestamp_3) - int(t0_system)
-
-            avg_graph_lat = \
-                    (int(graph_lat_1) +\
-                     int(graph_lat_2) +\
-                     int(graph_lat_3)) / 3
-    
-            if avg_graph_lat > sw_disc_threshold_min \
-                    and avg_graph_lat < sw_disc_threshold_max:
-                sw_discovery_lat_list.append(
-                        avg_graph_lat)
-            else:
-                main.log.info("100 Switch discovery latency "+
-                        "exceeded the threshold.")
-            
-            #END ITERATION FOR LOOP
-
-        sw_lat_min = min(sw_discovery_lat_list)
-        sw_lat_max = max(sw_discovery_lat_list)
-        sw_lat_avg = sum(sw_discovery_lat_list) /\
-                     len(sw_discovery_lat_list)
-
-        main.log.report("100 Switch discovery lat "+\
-                "Min: "+str(sw_lat_min)+" ms"+\
-                "Max: "+str(sw_lat_max)+" ms"+\
-                "Avg: "+str(sw_lat_avg)+" ms")
-
-    def CASE6(self, main):
-        '''
-        Increase number of nodes and initiate CLI
-        '''
-        import time
-        
-        ONOS1_ip = main.params['CTRL']['ip1']
-        ONOS2_ip = main.params['CTRL']['ip2']
-        ONOS3_ip = main.params['CTRL']['ip3']
-        ONOS4_ip = main.params['CTRL']['ip4']
-        ONOS5_ip = main.params['CTRL']['ip5']
-        ONOS6_ip = main.params['CTRL']['ip6']
-        ONOS7_ip = main.params['CTRL']['ip7']
-
-        cell_name = main.params['ENV']['cellName']
-    
-        global cluster_count
-        
-        #Cluster size increased everytime the case is defined
-        cluster_count += 2 
-
-        main.log.report("Increasing cluster size to "+
-                str(cluster_count))
-
-        install_result = main.FALSE
-        if cluster_count == 5:
-            main.log.info("Installing nodes 4 and 5")
-            node4_result = \
-                main.ONOSbench.onos_install(node=ONOS4_ip)
-            node5_result = \
-                main.ONOSbench.onos_install(node=ONOS5_ip)
-            install_result = node4_result and node5_result
-
-            time.sleep(5)
-
-            main.ONOS4cli.start_onos_cli(ONOS4_ip)
-            main.ONOS5cli.start_onos_cli(ONOS5_ip)
-
-        elif cluster_count == 7:
-            main.log.info("Installing nodes 4 and 5")
-            node6_result = \
-                main.ONOSbench.onos_install(node=ONOS6_ip)
-            node7_result = \
-                main.ONOSbench.onos_install(node=ONOS7_ip)
-            install_result = node6_result and node7_result
-
-            time.sleep(5)
-
-            main.ONOS6cli.start_onos_cli(ONOS6_ip)
-            main.ONOS7cli.start_onos_cli(ONOS7_ip)
-
-
-
-
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.py b/TestON/tests/TopoPerfNext/TopoPerfNext.py
deleted file mode 100644
index 0f37909..0000000
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.py
+++ /dev/null
@@ -1,2006 +0,0 @@
-# TopoPerfNext
-#
-# Topology Performance test for ONOS-next
-#
-# andrew@onlab.us
-#
-# If your machine does not come with numpy
-# run the following command:
-# sudo apt-get install python-numpy python-scipy
-
-import time
-import sys
-import os
-import re
-
-
-class TopoPerfNext:
-
-    def __init__( self ):
-        self.default = ''
-
-    def CASE1( self, main ):
-        """
-        ONOS startup sequence
-        """
-        import time
-
-        # Global cluster count for scale-out purposes
-        global clusterCount
-        #TODO: fix run number implementation
-        global runNum 
-        global timeToPost
-
-        #Test run time
-        timeToPost = time.strftime("%Y-%m-%d %H:%M:%S")
-        # Set initial cluster count
-        clusterCount = 1
-        ##
-
-        runNum = time.strftime("%d%H%M%S")
-
-        cellName = main.params[ 'ENV' ][ 'cellName' ]
-
-        gitPull = main.params[ 'GIT' ][ 'autoPull' ]
-        checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
-
-        ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
-        ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
-        ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
-        ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ] 
-        ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
-        ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
-        ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ] 
-
-        MN1Ip = main.params[ 'MN' ][ 'ip1' ]
-        BENCHIp = main.params[ 'BENCH' ][ 'ip' ]
-
-        topoCfgFile = main.params[ 'TEST' ][ 'topoConfigFile' ]
-        topoCfgName = main.params[ 'TEST' ][ 'topoConfigName' ]
-
-        portEventResultPath = main.params[ 'DB' ][ 'portEventResultPath' ]
-        switchEventResultPath = main.params[ 'DB' ][ 'switchEventResultPath' ]
-
-        mvnCleanInstall = main.params[ 'TEST' ][ 'mci' ]
-        
-        main.case( "Setting up test environment" )
-        main.log.info( "Copying topology event accumulator config" +
-                       " to ONOS /package/etc" )
-        main.ONOSbench.handle.sendline( "cp ~/" +
-                                        topoCfgFile +
-                                        " ~/ONOS/tools/package/etc/" +
-                                        topoCfgName )
-        main.ONOSbench.handle.expect( "\$" )
-
-        main.log.report( "Setting up test environment" )
-
-        main.step( "Starting mininet topology " )
-        main.Mininet1.startNet()
-
-        main.step( "Cleaning previously installed ONOS if any" )
-        main.ONOSbench.onosUninstall( nodeIp=ONOS2Ip )
-        main.ONOSbench.onosUninstall( nodeIp=ONOS3Ip )
-        main.ONOSbench.onosUninstall( nodeIp=ONOS4Ip )
-        main.ONOSbench.onosUninstall( nodeIp=ONOS5Ip )
-        main.ONOSbench.onosUninstall( nodeIp=ONOS6Ip )
-        main.ONOSbench.onosUninstall( nodeIp=ONOS7Ip )
-
-        main.step( "Clearing previous DB log files" )
-        fPortLog = open(portEventResultPath, 'w')
-        fPortLog.write('')
-        fPortLog.close()
-        fSwitchLog = open(switchEventResultPath, 'w')
-        fSwitchLog.write('')
-        fSwitchLog.close()
-        
-        main.step( "Creating cell file" )
-        cellFileResult = main.ONOSbench.createCellFile(
-            BENCHIp, cellName, MN1Ip,
-            ("onos-core,onos-api,webconsole,onos-app-metrics,onos-app-gui,"
-            "onos-cli,onos-openflow"),
-            ONOS1Ip )
-
-        main.step( "Applying cell file to environment" )
-        cellApplyResult = main.ONOSbench.setCell( cellName )
-        verifyCellResult = main.ONOSbench.verifyCell()
-
-        # NOTE: This step may be removed after proper
-        #      copy cat log functionality
-        main.step( "Removing raft/copy-cat logs from ONOS nodes" )
-        main.ONOSbench.onosRemoveRaftLogs()
-        time.sleep( 30 )
-
-        main.step( "Git checkout and pull " + checkoutBranch )
-        if gitPull == 'on':
-            # checkoutResult = \
-                    #        main.ONOSbench.gitCheckout( checkoutBranch )
-            checkoutResult = main.TRUE
-            pullResult = main.ONOSbench.gitPull()
-        else:
-            checkoutResult = main.TRUE
-            pullResult = main.TRUE
-            main.log.info( "Skipped git checkout and pull" )
-
-        main.log.report( "Commit information - " )
-        main.ONOSbench.getVersion( report=True )
-
-        main.step( "Using mvn clean & install" )
-        if mvnCleanInstall == 'on':
-            mvnResult = main.ONOSbench.cleanInstall()
-        elif mvnCleanInstall == 'off':
-            main.log.info("mci turned off by settings")
-            mvnResult = main.TRUE
-
-        main.step( "Set cell for ONOS cli env" )
-        main.ONOS1cli.setCell( cellName )
-
-        main.step( "Creating ONOS package" )
-        packageResult = main.ONOSbench.onosPackage()
-
-        main.step( "Installing ONOS package" )
-        install1Result = main.ONOSbench.onosInstall( node=ONOS1Ip )
-
-        time.sleep( 10 )
-
-        main.step( "Start onos cli" )
-        cli1 = main.ONOS1cli.startOnosCli( ONOS1Ip )
-
-        utilities.assert_equals( expect=main.TRUE,
-                                actual=cellFileResult and cellApplyResult and
-                                verifyCellResult and checkoutResult and
-                                pullResult and mvnResult and
-                                install1Result,  # and install2Result and
-                                # install3Result,
-                                onpass="Test Environment setup successful",
-                                onfail="Failed to setup test environment" )
-
-    def CASE2( self, main ):
-        """
-        Assign s1 to ONOS1 and measure latency
-
-        There are 4 levels of latency measurements to this test:
-        1 ) End-to-end measurement: Complete end-to-end measurement
-           from TCP ( SYN/ACK ) handshake to Graph change
-        2 ) OFP-to-graph measurement: 'ONOS processing' snippet of
-           measurement from OFP Vendor message to Graph change
-        3 ) OFP-to-device measurement: 'ONOS processing without
-           graph change' snippet of measurement from OFP vendor
-           message to Device change timestamp
-        4 ) T0-to-device measurement: Measurement that includes
-           the switch handshake to devices timestamp without
-           the graph view change. ( TCP handshake -> Device
-           change )
-        """
-        import time
-        import subprocess
-        import json
-        import requests
-        import os
-        import numpy
-        global clusterCount
-        global timeToPost
-
-        ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
-        ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
-        ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
-        ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
-        ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
-        ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
-        ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
-
-        ONOSUser = main.params[ 'CTRL' ][ 'user' ]
-
-        defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
-
-        # Number of iterations of case
-        numIter = main.params[ 'TEST' ][ 'numIter' ]
-        # Number of first 'x' iterations to ignore:
-        iterIgnore = int( main.params[ 'TEST' ][ 'iterIgnore' ] )
-
-        # Timestamp 'keys' for json metrics output.
-        # These are subject to change, hence moved into params
-        deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
-        graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
-
-        debugMode = main.params[ 'TEST' ][ 'debugMode' ]
-        onosLog = main.params[ 'TEST' ][ 'onosLogFile' ]
-        resultPath = main.params[ 'DB' ][ 'switchEventResultPath' ]
-
-        # Threshold for the test
-        thresholdStr = main.params[ 'TEST' ][ 'singleSwThreshold' ]
-        thresholdObj = thresholdStr.split( "," )
-        thresholdMin = int( thresholdObj[ 0 ] )
-        thresholdMax = int( thresholdObj[ 1 ] )
-
-        # List of switch add latency collected from
-        # all iterations
-        latencyEndToEndList = []
-        latencyOfpToGraphList = []
-        latencyOfpToDeviceList = []
-        latencyT0ToDeviceList = []
-        latencyTcpToOfpList = []
-
-        # Initialize 2d array for [node][iteration] storage
-        endToEndLatNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
-        ofpToGraphLatNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
-        # tcp-to-ofp measurements are same throughout each iteration 
-        tcpToOfpLatIter = [] 
-
-        # Directory/file to store tshark results
-        tsharkOfOutput = "/tmp/tshark_of_topo.txt"
-        tsharkTcpOutput = "/tmp/tshark_tcp_topo.txt"
-
-        # String to grep in tshark output
-        tsharkTcpString = "TCP 74 " + defaultSwPort
-        tsharkOfString = "OFP 86 Vendor"
-
-        # Initialize assertion to TRUE
-        assertion = main.TRUE
-
-        localTime = time.strftime( '%x %X' )
-        localTime = localTime.replace( "/", "" )
-        localTime = localTime.replace( " ", "_" )
-        localTime = localTime.replace( ":", "" )
-        if debugMode == 'on':
-            main.ONOS1.tsharkPcap( "eth0",
-                                   "/tmp/single_sw_lat_pcap_" + localTime )
-
-            main.log.info( "Debug mode is on" )
-
-        main.log.report( "Latency of adding one switch to controller" )
-        main.log.report( "First " + str( iterIgnore ) + " iterations ignored" +
-                         " for jvm warmup time" )
-        main.log.report( "Total iterations of test: " + str( numIter ) )
-
-        for i in range( 0, int( numIter ) ):
-            main.log.info( "Starting tshark capture" )
-
-            #* TCP [ ACK, SYN ] is used as t0A, the
-            #  very first "exchange" between ONOS and
-            #  the switch for end-to-end measurement
-            #* OFP [ Stats Reply ] is used for t0B
-            #  the very last OFP message between ONOS
-            #  and the switch for ONOS measurement
-            main.ONOS1.tsharkGrep( tsharkTcpString,
-                                   tsharkTcpOutput )
-            main.ONOS1.tsharkGrep( tsharkOfString,
-                                   tsharkOfOutput )
-
-            # Wait and ensure tshark is started and
-            # capturing
-            time.sleep( 10 )
-
-            main.log.info( "Assigning s1 to controller" )
-
-            main.Mininet1.assignSwController(
-                sw="1",
-                ip1=ONOS1Ip,
-                port1=defaultSwPort )
-
-            # Wait and ensure switch is assigned
-            # before stopping tshark
-            time.sleep( 30 )
-
-            main.log.info( "Stopping all Tshark processes" )
-            main.ONOS1.stopTshark()
-
-            # tshark output is saved in ONOS. Use subprocess
-            # to copy over files to TestON for parsing
-            main.log.info( "Copying over tshark files" )
-
-            # TCP CAPTURE ****
-            # Copy the tshark output from ONOS machine to
-            # TestON machine in tsharkTcpOutput directory>file
-            os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
-                       tsharkTcpOutput + " /tmp/" )
-            tcpFile = open( tsharkTcpOutput, 'r' )
-            tempText = tcpFile.readline()
-            tempText = tempText.split( " " )
-
-            main.log.info( "Object read in from TCP capture: " +
-                           str( tempText ) )
-            if len( tempText ) > 1:
-                t0Tcp = float( tempText[ 1 ] ) * 1000.0
-            else:
-                main.log.error( "Tshark output file for TCP" +
-                                " returned unexpected results" )
-                t0Tcp = 0
-                assertion = main.FALSE
-
-            tcpFile.close()
-            #****************
-
-            # OF CAPTURE ****
-            os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
-                       tsharkOfOutput + " /tmp/" )
-            ofFile = open( tsharkOfOutput, 'r' )
-
-            lineOfp = ""
-            # Read until last line of file
-            while True:
-                tempText = ofFile.readline()
-                if tempText != '':
-                    lineOfp = tempText
-                else:
-                    break
-            obj = lineOfp.split( " " )
-
-            main.log.info( "Object read in from OFP capture: " +
-                           str( lineOfp ) )
-
-            if len( obj ) > 1:
-                t0Ofp = float( obj[ 1 ] ) * 1000.0
-            else:
-                main.log.error( "Tshark output file for OFP" +
-                                " returned unexpected results" )
-                t0Ofp = 0
-                assertion = main.FALSE
-
-            ofFile.close()
-            #****************
-
-            jsonStr1 = main.ONOS1cli.topologyEventsMetrics()
-            # Initialize scale-out variables
-            jsonStr2 = ""
-            jsonStr3 = ""
-            jsonStr4 = ""
-            jsonStr5 = ""
-            jsonStr6 = ""
-            jsonStr7 = ""
-
-            jsonObj1 = json.loads( jsonStr1 )
-            # Initialize scale-out variables
-            jsonObj2 = ""
-            jsonObj3 = ""
-            jsonObj4 = ""
-            jsonObj5 = ""
-            jsonObj6 = ""
-            jsonObj7 = ""
-
-            # Obtain graph timestamp. This timestsamp captures
-            # the epoch time at which the topology graph was updated.
-            graphTimestamp1 = \
-                jsonObj1[ graphTimestamp ][ 'value' ]
-            # Obtain device timestamp. This timestamp captures
-            # the epoch time at which the device event happened
-            deviceTimestamp1 = \
-                jsonObj1[ deviceTimestamp ][ 'value' ]
-
-            # t0 to device processing latency
-            deltaDevice1 = int( deviceTimestamp1 ) - int( t0Tcp )
-            # t0 to graph processing latency ( end-to-end )
-            deltaGraph1 = int( graphTimestamp1 ) - int( t0Tcp )
-            # ofp to graph processing latency ( ONOS processing )
-            deltaOfpGraph1 = int( graphTimestamp1 ) - int( t0Ofp )
-            # ofp to device processing latency ( ONOS processing )
-            deltaOfpDevice1 = float( deviceTimestamp1 ) - float( t0Ofp )
-            # tcp to ofp processing latency ( switch connection )
-            deltaTcpOfp1 = int(t0Ofp) - int(t0Tcp)
-
-            if deltaTcpOfp1 > thresholdMin and deltaTcpOfp1 < thresholdMax\
-               and i >= iterIgnore:
-                tcpToOfpLatIter.append(deltaTcpOfp1)
-                main.log.info("iter"+str(i)+" tcp-to-ofp: "+
-                          str(deltaTcpOfp1)+" ms")
-            else:
-                tcpToOfpLatIter.append(0)
-                main.log.info("iter"+str(i)+" tcp-to-ofp: "+
-                          str(deltaTcpOfp1)+" ms - ignored this iteration")
-
-            # Store initial measurements in data array
-            #This measurement is for node 1
-           
-            if deltaGraph1 > thresholdMin and deltaGraph1 < thresholdMax\
-               and i >= iterIgnore:
-                endToEndLatNodeIter[0][i] = deltaGraph1
-                main.log.info("ONOS1 iter"+str(i)+" end-to-end: "+
-                          str(deltaGraph1)+" ms")
-            else:
-                main.log.info("ONOS1 iter"+str(i)+" end-to-end: "+
-                          str(deltaGraph1)+" ms - ignored this iteration")
-
-
-            if deltaOfpGraph1 > thresholdMin and deltaOfpGraph1 < thresholdMax\
-               and i >= iterIgnore:
-                ofpToGraphLatNodeIter[0][i] = deltaOfpGraph1
-             
-            main.log.info("ONOS1 iter"+str(i)+" ofp-to-graph: "+
-                          str(deltaOfpGraph1)+" ms")
-
-            # TODO: Create even cluster number events
-
-            # Include scale-out measurements when applicable
-            if clusterCount >= 3:
-                jsonStr2 = main.ONOS2cli.topologyEventsMetrics()
-                jsonStr3 = main.ONOS3cli.topologyEventsMetrics()
-                jsonObj2 = json.loads( jsonStr2 )
-                jsonObj3 = json.loads( jsonStr3 )
-                graphTimestamp2 = \
-                    jsonObj2[ graphTimestamp ][ 'value' ]
-                graphTimestamp3 = \
-                    jsonObj3[ graphTimestamp ][ 'value' ]
-                deviceTimestamp2 = \
-                    jsonObj2[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp3 = \
-                    jsonObj3[ deviceTimestamp ][ 'value' ]
-                deltaDevice2 = int( deviceTimestamp2 ) - int( t0Tcp )
-                deltaDevice3 = int( deviceTimestamp3 ) - int( t0Tcp )
-                deltaGraph2 = int( graphTimestamp2 ) - int( t0Tcp )
-                deltaGraph3 = int( graphTimestamp3 ) - int( t0Tcp )
-                deltaOfpGraph2 = int( graphTimestamp2 ) - int( t0Ofp )
-                deltaOfpGraph3 = int( graphTimestamp3 ) - int( t0Ofp )
-                deltaOfpDevice2 = float( deviceTimestamp2 ) -\
-                    float( t0Ofp )
-                deltaOfpDevice3 = float( deviceTimestamp3 ) -\
-                    float( t0Ofp )
-
-                if deltaGraph2 > thresholdMin and\
-                   deltaGraph2 < thresholdMax and i >= iterIgnore:
-                    endToEndLatNodeIter[1][i] = deltaGraph2
-                    main.log.info("ONOS2 iter"+str(i)+" end-to-end: "+
-                            str(deltaGraph2)+" ms")
-                
-                if deltaOfpGraph2 > thresholdMin and\
-                   deltaOfpGraph2 < thresholdMax and i >= iterIgnore:
-                    ofpToGraphLatNodeIter[1][i] = deltaOfpGraph2
-                    main.log.info("ONOS2 iter"+str(i)+" ofp-to-graph: "+
-                            str(deltaOfpGraph2)+" ms")
-
-                if deltaGraph3 > thresholdMin and\
-                   deltaGraph3 < thresholdMax and i >= iterIgnore:
-                    endToEndLatNodeIter[2][i] = deltaGraph3
-                    main.log.info("ONOS3 iter"+str(i)+" end-to-end: "+
-                            str(deltaGraph3)+" ms")
-                
-                if deltaOfpGraph3 > thresholdMin and\
-                   deltaOfpGraph3 < thresholdMax and i >= iterIgnore:
-                    ofpToGraphLatNodeIter[2][i] = deltaOfpGraph3
-                    main.log.info("ONOS3 iter"+str(i)+" ofp-to-graph: "+
-                            str(deltaOfpGraph3)+" ms")
-                
-            if clusterCount >= 5:
-                jsonStr4 = main.ONOS4cli.topologyEventsMetrics()
-                jsonStr5 = main.ONOS5cli.topologyEventsMetrics()
-                jsonObj4 = json.loads( jsonStr4 )
-                jsonObj5 = json.loads( jsonStr5 )
-                graphTimestamp4 = \
-                    jsonObj4[ graphTimestamp ][ 'value' ]
-                graphTimestamp5 = \
-                    jsonObj5[ graphTimestamp ][ 'value' ]
-                deviceTimestamp4 = \
-                    jsonObj4[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp5 = \
-                    jsonObj5[ deviceTimestamp ][ 'value' ]
-                deltaDevice4 = int( deviceTimestamp4 ) - int( t0Tcp )
-                deltaDevice5 = int( deviceTimestamp5 ) - int( t0Tcp )
-                deltaGraph4 = int( graphTimestamp4 ) - int( t0Tcp )
-                deltaGraph5 = int( graphTimestamp5 ) - int( t0Tcp )
-                deltaOfpGraph4 = int( graphTimestamp4 ) - int( t0Ofp )
-                deltaOfpGraph5 = int( graphTimestamp5 ) - int( t0Ofp )
-                deltaOfpDevice4 = float( deviceTimestamp4 ) -\
-                    float( t0Ofp )
-                deltaOfpDevice5 = float( deviceTimestamp5 ) -\
-                    float( t0Ofp )
-                
-                if deltaGraph4 > thresholdMin and\
-                   deltaGraph4 < thresholdMax and i >= iterIgnore:
-                    endToEndLatNodeIter[3][i] = deltaGraph4
-                    main.log.info("ONOS4 iter"+str(i)+" end-to-end: "+
-                            str(deltaGraph4)+" ms")
-                
-                    #TODO:
-                if deltaOfpGraph4 > thresholdMin and\
-                   deltaOfpGraph4 < thresholdMax and i >= iterIgnore:
-                    ofpToGraphLatNodeIter[3][i] = deltaOfpGraph4
-                    main.log.info("ONOS4 iter"+str(i)+" ofp-to-graph: "+
-                            str(deltaOfpGraph4)+" ms")
-
-                if deltaGraph5 > thresholdMin and\
-                   deltaGraph5 < thresholdMax and i >= iterIgnore:
-                    endToEndLatNodeIter[4][i] = deltaGraph5
-                    main.log.info("ONOS5 iter"+str(i)+" end-to-end: "+
-                            str(deltaGraph5)+" ms")
-                
-                if deltaOfpGraph5 > thresholdMin and\
-                   deltaOfpGraph5 < thresholdMax and i >= iterIgnore:
-                    ofpToGraphLatNodeIter[4][i] = deltaOfpGraph5
-                    main.log.info("ONOS5 iter"+str(i)+" ofp-to-graph: "+
-                            str(deltaOfpGraph5)+" ms")
-                
-            if clusterCount >= 7:
-                jsonStr6 = main.ONOS6cli.topologyEventsMetrics()
-                jsonStr7 = main.ONOS7cli.topologyEventsMetrics()
-                jsonObj6 = json.loads( jsonStr6 )
-                jsonObj7 = json.loads( jsonStr7 )
-                graphTimestamp6 = \
-                    jsonObj6[ graphTimestamp ][ 'value' ]
-                graphTimestamp7 = \
-                    jsonObj7[ graphTimestamp ][ 'value' ]
-                deviceTimestamp6 = \
-                    jsonObj6[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp7 = \
-                    jsonObj7[ deviceTimestamp ][ 'value' ]
-                deltaDevice6 = int( deviceTimestamp6 ) - int( t0Tcp )
-                deltaDevice7 = int( deviceTimestamp7 ) - int( t0Tcp )
-                deltaGraph6 = int( graphTimestamp6 ) - int( t0Tcp )
-                deltaGraph7 = int( graphTimestamp7 ) - int( t0Tcp )
-                deltaOfpGraph6 = int( graphTimestamp6 ) - int( t0Ofp )
-                deltaOfpGraph7 = int( graphTimestamp7 ) - int( t0Ofp )
-                deltaOfpDevice6 = float( deviceTimestamp6 ) -\
-                    float( t0Ofp )
-                deltaOfpDevice7 = float( deviceTimestamp7 ) -\
-                    float( t0Ofp )
-                
-                if deltaGraph6 > thresholdMin and\
-                   deltaGraph6 < thresholdMax and i >= iterIgnore:
-                    endToEndLatNodeIter[5][i] = deltaGraph6
-                    main.log.info("ONOS6 iter"+str(i)+" end-to-end: "+
-                            str(deltaGraph6)+" ms")
-                
-                    #TODO:
-                if deltaOfpGraph6 > thresholdMin and\
-                   deltaOfpGraph6 < thresholdMax and i >= iterIgnore:
-                    ofpToGraphLatNodeIter[5][i] = deltaOfpGraph6
-                    main.log.info("ONOS6 iter"+str(i)+" ofp-to-graph: "+
-                            str(deltaOfpGraph6)+" ms")
-
-                if deltaGraph7 > thresholdMin and\
-                   deltaGraph7 < thresholdMax and i >= iterIgnore:
-                    endToEndLatNodeIter[6][i] = deltaGraph7
-                    main.log.info("ONOS7 iter"+str(i)+" end-to-end: "+
-                            str(deltaGraph7)+" ms")
-                
-                if deltaOfpGraph7 > thresholdMin and\
-                   deltaOfpGraph7 < thresholdMax and i >= iterIgnore:
-                    ofpToGraphLatNodeIter[6][i] = deltaOfpGraph7
-                    main.log.info("ONOS7 iter"+str(i)+" ofp-to-graph: "+
-                            str(deltaOfpGraph7)+" ms")
-                    
-            main.log.info("Switch up discovery latency")
-                
-            main.log.info("Starting tshark capture")
-
-            main.step( "Remove switch from controller" )
-            main.Mininet1.deleteSwController( "s1" )
-
-            #TODO: del controller does not have an OFP message.
-            #      However, we can capture TCP Fin,Ack as T0
-
-            time.sleep( 5 )
-
-        # END of for loop iteration
-
-        #str( round( numpy.std( latencyT0ToDeviceList ), 1 ) )
-
-        endToEndAvg = 0
-        ofpToGraphAvg = 0
-        endToEndList = []
-        ofpToGraphList = []
-        dbCmdList = []
-
-        for node in range( 0, clusterCount ):
-            # The latency 2d array was initialized to 0. 
-            # If an iteration was ignored, then we have some 0's in
-            # our calculation. To avoid having this interfere with our 
-            # results, we must delete any index where 0 is found...
-            # WARN: Potentially, we could have latency that hovers at
-            # 0 ms once we have optimized code. FIXME for when this is
-            # the case. Being able to obtain sub-millisecond accuracy
-            # can prevent this from happening
-            for item in endToEndLatNodeIter[node]:
-                if item > 0.0:
-                    endToEndList.append(item)
-            for item in ofpToGraphLatNodeIter[node]:
-                if item > 0.0:
-                    ofpToGraphList.append(item)
-
-            endToEndAvg = round(numpy.mean(endToEndList), 2)
-            ofpToGraphAvg = numpy.mean(ofpToGraphList)
-            endToEndStd = round(numpy.std(endToEndList), 2)
-
-            main.log.report( " - Node "+str(node+1)+" Summary - " )
-            main.log.report( " End-to-end Avg: "+
-                             str(round(endToEndAvg,2))+" ms"+
-                             " End-to-end Std dev: "+
-                             str(round(endToEndStd,2))+" ms")
-            
-            dbCmdList.append(
-                "INSERT INTO switch_latency_tests VALUES("
-                   "'"+timeToPost+"','switch_latency_results',"
-                   ""+runNum+","+str(clusterCount)+",'baremetal"+str(node+1)+"',"
-                   ""+str(endToEndAvg)+","+str(endToEndStd)+",0,0);"
-            )
-            #main.log.report( " Ofp-to-graph Avg: "+
-            #                 str(round(ofpToGraphAvg,2))+" ms"+
-            #                 " Ofp-to-graph Std dev: "+
-            #                 str(round(numpy.std(ofpToGraphList),2))+
-            #                 " ms")
-
-        if debugMode == 'on':
-            main.ONOS1.cpLogsToDir( "/opt/onos/log/karaf.log",
-                                      "/tmp/", copyFileName="sw_lat_karaf" )
-        
-        #Write to file for posting to DB
-        fResult = open(resultPath, 'a')
-        for line in dbCmdList:
-            if line:
-                fResult.write(line+"\n")
-        fResult.close()
-
-        #TODO: correct assert
-        assertion = main.TRUE
-
-        utilities.assert_equals( expect=main.TRUE, actual=assertion,
-                                onpass="Switch latency test successful",
-                                onfail="Switch latency test failed" )
-
-    def CASE3( self, main ):
-        """
-        Bring port up / down and measure latency.
-        Port enable / disable is simulated by ifconfig up / down
-
-        In ONOS-next, we must ensure that the port we are
-        manipulating is connected to another switch with a valid
-        connection. Otherwise, graph view will not be updated.
-        """
-        import time
-        import subprocess
-        import os
-        import requests
-        import json
-        import numpy
-        global clusterCount
-        global runNum
-        global timeToPost
-
-        ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
-        ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
-        ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
-        ONOSUser = main.params[ 'CTRL' ][ 'user' ]
-
-        defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
-
-        assertion = main.TRUE
-        # Number of iterations of case
-        numIter = main.params[ 'TEST' ][ 'numIter' ]
-        iterIgnore = int( main.params[ 'TEST' ][ 'iterIgnore' ] )
-
-        # Timestamp 'keys' for json metrics output.
-        # These are subject to change, hence moved into params
-        deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
-        graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
-
-        debugMode = main.params[ 'TEST' ][ 'debugMode' ]
-        postToDB = main.params[ 'DB' ][ 'postToDB' ]
-        resultPath = main.params[ 'DB' ][ 'portEventResultPath' ]
-
-        localTime = time.strftime( '%x %X' )
-        localTime = localTime.replace( "/", "" )
-        localTime = localTime.replace( " ", "_" )
-        localTime = localTime.replace( ":", "" )
-        if debugMode == 'on':
-            main.ONOS1.tsharkPcap( "eth0",
-                                   "/tmp/port_lat_pcap_" + localTime )
-
-        # Threshold for this test case
-        upThresholdStr = main.params[ 'TEST' ][ 'portUpThreshold' ]
-        downThresholdStr = main.params[ 'TEST' ][ 'portDownThreshold' ]
-
-        upThresholdObj = upThresholdStr.split( "," )
-        downThresholdObj = downThresholdStr.split( "," )
-
-        upThresholdMin = int( upThresholdObj[ 0 ] )
-        upThresholdMax = int( upThresholdObj[ 1 ] )
-
-        downThresholdMin = int( downThresholdObj[ 0 ] )
-        downThresholdMax = int( downThresholdObj[ 1 ] )
-
-        # NOTE: Some hardcoded variables you may need to configure
-        #      besides the params
-
-        tsharkPortStatus = "OFP 130 Port Status"
-
-        tsharkPortUp = "/tmp/tshark_port_up.txt"
-        tsharkPortDown = "/tmp/tshark_port_down.txt"
-        interfaceConfig = "s1-eth1"
-
-        main.log.report( "Port enable / disable latency" )
-        main.log.report( "Simulated by ifconfig up / down" )
-        main.log.report( "Total iterations of test: " + str( numIter ) )
-
-        main.step( "Assign switches s1 and s2 to controller 1" )
-        main.Mininet1.assignSwController( sw="1", ip1=ONOS1Ip,
-                                           port1=defaultSwPort )
-        main.Mininet1.assignSwController( sw="2", ip1=ONOS1Ip,
-                                           port1=defaultSwPort )
-
-        # Give enough time for metrics to propagate the
-        # assign controller event. Otherwise, these events may
-        # carry over to our measurements
-        time.sleep( 15 )
-
-        portUpDeviceToOfpList = []
-        portUpGraphToOfpList = []
-        portDownDeviceToOfpList = []
-        portDownGraphToOfpList = []
-       
-        # Initialize 2d array filled with 0's
-        # arraySizeFormat[clusterCount][numIter]
-        portUpDevNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
-        portUpGraphNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
-        portDownDevNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
-        portDownGraphNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
-
-        for i in range( 0, int( numIter ) ):
-            main.step( "Starting wireshark capture for port status down" )
-            main.ONOS1.tsharkGrep( tsharkPortStatus,
-                                   tsharkPortDown )
-
-            time.sleep( 5 )
-
-            # Disable interface that is connected to switch 2
-            main.step( "Disable port: " + interfaceConfig )
-            main.Mininet1.handle.sendline( "sh ifconfig " +
-                                           interfaceConfig + " down" )
-            main.Mininet1.handle.expect( "mininet>" )
-
-            time.sleep( 3 )
-            main.ONOS1.tsharkStop()
-
-            # Copy tshark output file from ONOS to TestON instance
-            #/tmp directory
-            os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
-                       tsharkPortDown + " /tmp/" )
-
-            fPortDown = open( tsharkPortDown, 'r' )
-            # Get first line of port down event from tshark
-            fLine = fPortDown.readline()
-            objDown = fLine.split( " " )
-            if len( fLine ) > 0:
-                # NOTE: objDown[ 1 ] is a very unreliable
-                #      way to determine the timestamp. If
-                #      results seem way off, check the object
-                #      itself by printing it out
-                timestampBeginPtDown = int( float( objDown[ 1 ] ) * 1000 )
-                # For some reason, wireshark decides to record the
-                # timestamp at the 3rd object position instead of
-                # 2nd at unpredictable times. This statement is
-                # used to capture that odd behavior and use the
-                # correct epoch time
-                if timestampBeginPtDown < 1400000000000:
-                    timestampBeginPtDown = \
-                        int( float( objDown[ 2 ] ) * 1000 )
-
-                main.log.info( "Port down begin timestamp: " +
-                               str( timestampBeginPtDown ) )
-            else:
-                main.log.info( "Tshark output file returned unexpected" +
-                               " results: " + str( objDown ) )
-                timestampBeginPtDown = 0
-            fPortDown.close()
-
-            main.step( "Obtain t1 by metrics call" )
-            jsonStrUp1 = main.ONOS1cli.topologyEventsMetrics()
-            jsonObj1 = json.loads( jsonStrUp1 )
-            # Obtain graph timestamp. This timestsamp captures
-            # the epoch time at which the topology graph was updated.
-            graphTimestamp1 = \
-                jsonObj1[ graphTimestamp ][ 'value' ]
-            # Obtain device timestamp. This timestamp captures
-            # the epoch time at which the device event happened
-            deviceTimestamp1 = \
-                jsonObj1[ deviceTimestamp ][ 'value' ]
-            # Get delta between graph event and OFP
-            ptDownGraphToOfp1 = int( graphTimestamp1 ) -\
-                int( timestampBeginPtDown )
-            # Get delta between device event and OFP
-            ptDownDeviceToOfp1 = int( deviceTimestamp1 ) -\
-                int( timestampBeginPtDown )
-
-            if ptDownGraphToOfp1 > downThresholdMin and\
-               ptDownGraphToOfp1 < downThresholdMax and i > iterIgnore:
-                portDownGraphNodeIter[0][i] = ptDownGraphToOfp1
-                main.log.info("iter"+str(i)+" port down graph-to-ofp: "+
-                              str(ptDownGraphToOfp1)+" ms")
-            else:
-                main.log.info("iter"+str(i)+" skipped. Result: "+
-                              str(ptDownGraphToOfp1)+" ms")
-            if ptDownDeviceToOfp1 > downThresholdMin and\
-               ptDownDeviceToOfp1 < downThresholdMax and i > iterIgnore:
-                portDownDevNodeIter[0][i] = ptDownDeviceToOfp1
-                main.log.info("iter"+str(i)+" port down device-to-ofp: "+
-                              str(ptDownDeviceToOfp1)+" ms")
-            else:
-                main.log.info("iter"+str(i)+" skipped. Result: "+
-                              str(ptDownDeviceToOfp1)+" ms")
-
-            if clusterCount >= 3:
-                jsonStrUp2 = main.ONOS2cli.topologyEventsMetrics()
-                jsonStrUp3 = main.ONOS3cli.topologyEventsMetrics()
-                jsonObj2 = json.loads( jsonStrUp2 )
-                jsonObj3 = json.loads( jsonStrUp3 )
-                graphTimestamp2 = \
-                    jsonObj2[ graphTimestamp ][ 'value' ]
-                graphTimestamp3 = \
-                    jsonObj3[ graphTimestamp ][ 'value' ]
-                deviceTimestamp2 = \
-                    jsonObj2[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp3 = \
-                    jsonObj3[ deviceTimestamp ][ 'value' ]
-                ptDownGraphToOfp2 = int( graphTimestamp2 ) -\
-                    int( timestampBeginPtDown )
-                ptDownGraphToOfp3 = int( graphTimestamp3 ) -\
-                    int( timestampBeginPtDown )
-                ptDownDeviceToOfp2 = int( deviceTimestamp2 ) -\
-                    int( timestampBeginPtDown )
-                ptDownDeviceToOfp3 = int( deviceTimestamp3 ) -\
-                    int( timestampBeginPtDown )
-
-                if ptDownGraphToOfp2 > downThresholdMin and\
-                   ptDownGraphToOfp2 < downThresholdMax and i > iterIgnore:
-                    portDownGraphNodeIter[1][i] = ptDownGraphToOfp2
-                    main.log.info("ONOS2 iter"+str(i)+" graph-to-ofp: "+
-                                  str(ptDownGraphToOfp2)+" ms")
-
-                if ptDownDeviceToOfp2 > downThresholdMin and\
-                   ptDownDeviceToOfp2 < downThresholdMax and i > iterIgnore:
-                    portDownDevNodeIter[1][i] = ptDownDeviceToOfp2
-                    main.log.info("ONOS2 iter"+str(i)+" device-to-ofp: "+
-                                  str(ptDownDeviceToOfp2)+" ms")
-
-                if ptDownGraphToOfp3 > downThresholdMin and\
-                   ptDownGraphToOfp3 < downThresholdMax and i > iterIgnore:
-                    portDownGraphNodeIter[2][i] = ptDownGraphToOfp3
-                    main.log.info("ONOS3 iter"+str(i)+" graph-to-ofp: "+
-                                  str(ptDownGraphToOfp3)+" ms")
-
-                if ptDownDeviceToOfp3 > downThresholdMin and\
-                   ptDownDeviceToOfp3 < downThresholdMax and i > iterIgnore:
-                    portDownDevNodeIter[2][i] = ptDownDeviceToOfp3
-                    main.log.info("ONOS3 iter"+str(i)+" device-to-ofp: "+
-                                  str(ptDownDeviceToOfp3)+" ms")
-
-            if clusterCount >= 5:
-                jsonStrUp4 = main.ONOS4cli.topologyEventsMetrics()
-                jsonStrUp5 = main.ONOS5cli.topologyEventsMetrics()
-                jsonObj4 = json.loads( jsonStrUp4 )
-                jsonObj5 = json.loads( jsonStrUp5 )
-                graphTimestamp4 = \
-                    jsonObj4[ graphTimestamp ][ 'value' ]
-                graphTimestamp5 = \
-                    jsonObj5[ graphTimestamp ][ 'value' ]
-                deviceTimestamp4 = \
-                    jsonObj4[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp5 = \
-                    jsonObj5[ deviceTimestamp ][ 'value' ]
-                ptDownGraphToOfp4 = int( graphTimestamp4 ) -\
-                    int( timestampBeginPtDown )
-                ptDownGraphToOfp5 = int( graphTimestamp5 ) -\
-                    int( timestampBeginPtDown )
-                ptDownDeviceToOfp4 = int( deviceTimestamp4 ) -\
-                    int( timestampBeginPtDown )
-                ptDownDeviceToOfp5 = int( deviceTimestamp5 ) -\
-                    int( timestampBeginPtDown )
-                
-                if ptDownGraphToOfp4 > downThresholdMin and\
-                   ptDownGraphToOfp4 < downThresholdMax and i > iterIgnore:
-                    portDownGraphNodeIter[3][i] = ptDownGraphToOfp4
-                    main.log.info("ONOS4 iter"+str(i)+" graph-to-ofp: "+
-                                  str(ptDownGraphToOfp4)+" ms")
-
-                if ptDownDeviceToOfp4 > downThresholdMin and\
-                   ptDownDeviceToOfp4 < downThresholdMax and i > iterIgnore:
-                    portDownDevNodeIter[3][i] = ptDownDeviceToOfp4
-                    main.log.info("ONOS4 iter"+str(i)+" device-to-ofp: "+
-                                  str(ptDownDeviceToOfp4)+" ms")
-
-                if ptDownGraphToOfp5 > downThresholdMin and\
-                   ptDownGraphToOfp5 < downThresholdMax and i > iterIgnore:
-                    portDownGraphNodeIter[4][i] = ptDownGraphToOfp5
-                    main.log.info("ONOS5 iter"+str(i)+" graph-to-ofp: "+
-                                  str(ptDownGraphToOfp5)+" ms")
-                
-                if ptDownDeviceToOfp5 > downThresholdMin and\
-                   ptDownDeviceToOfp5 < downThresholdMax and i > iterIgnore:
-                    portDownDevNodeIter[4][i] = ptDownDeviceToOfp5
-                    main.log.info("ONOS5 iter"+str(i)+" device-to-ofp: "+
-                                  str(ptDownDeviceToOfp5)+" ms")
-
-            if clusterCount >= 7:
-                jsonStrUp6 = main.ONOS6cli.topologyEventsMetrics()
-                jsonStrUp7 = main.ONOS7cli.topologyEventsMetrics()
-                jsonObj6 = json.loads( jsonStrUp6 )
-                jsonObj7 = json.loads( jsonStrUp7 )
-                graphTimestamp6 = \
-                    jsonObj6[ graphTimestamp ][ 'value' ]
-                graphTimestamp7 = \
-                    jsonObj7[ graphTimestamp ][ 'value' ]
-                deviceTimestamp6 = \
-                    jsonObj6[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp7 = \
-                    jsonObj7[ deviceTimestamp ][ 'value' ]
-                ptDownGraphToOfp6 = int( graphTimestamp6 ) -\
-                    int( timestampBeginPtDown )
-                ptDownGraphToOfp7 = int( graphTimestamp7 ) -\
-                    int( timestampBeginPtDown )
-                ptDownDeviceToOfp6 = int( deviceTimestamp6 ) -\
-                    int( timestampBeginPtDown )
-                ptDownDeviceToOfp7 = int( deviceTimestamp7 ) -\
-                    int( timestampBeginPtDown )
-                
-                if ptDownGraphToOfp6 > downThresholdMin and\
-                   ptDownGraphToOfp6 < downThresholdMax and i > iterIgnore:
-                    portDownGraphNodeIter[5][i] = ptDownGraphToOfp6
-                    main.log.info("ONOS6 iter"+str(i)+" graph-to-ofp: "+
-                                  str(ptDownGraphToOfp6)+" ms")
-
-                if ptDownDeviceToOfp6 > downThresholdMin and\
-                   ptDownDeviceToOfp6 < downThresholdMax and i > iterIgnore:
-                    portDownDevNodeIter[5][i] = ptDownDeviceToOfp6
-                    main.log.info("ONOS6 iter"+str(i)+" device-to-ofp: "+
-                                  str(ptDownDeviceToOfp6)+" ms")
-
-                if ptDownGraphToOfp7 > downThresholdMin and\
-                   ptDownGraphToOfp7 < downThresholdMax and i > iterIgnore:
-                    portDownGraphNodeIter[6][i] = ptDownGraphToOfp7
-                    main.log.info("ONOS7 iter"+str(i)+" graph-to-ofp: "+
-                                  str(ptDownGraphToOfp7)+" ms")
-                
-                if ptDownDeviceToOfp7 > downThresholdMin and\
-                   ptDownDeviceToOfp7 < downThresholdMax and i > iterIgnore:
-                    portDownDevNodeIter[6][i] = ptDownDeviceToOfp7
-                    main.log.info("ONOS7 iter"+str(i)+" device-to-ofp: "+
-                                  str(ptDownDeviceToOfp7)+" ms")
-
-            time.sleep( 3 )
-
-            # Port up events
-            main.step( "Enable port and obtain timestamp" )
-            main.step( "Starting wireshark capture for port status up" )
-            main.ONOS1.tsharkGrep( tsharkPortStatus, tsharkPortUp )
-            time.sleep( 5 )
-
-            main.Mininet1.handle.sendline( "sh ifconfig " +
-                                           interfaceConfig + " up" )
-            main.Mininet1.handle.expect( "mininet>" )
-
-            # Allow time for tshark to capture event
-            time.sleep( 5 )
-            main.ONOS1.tsharkStop()
-
-            time.sleep( 3 )
-            os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
-                       tsharkPortUp + " /tmp/" )
-            fPortUp = open( tsharkPortUp, 'r' )
-            fLine = fPortUp.readline()
-            objUp = fLine.split( " " )
-            if len( fLine ) > 0:
-                timestampBeginPtUp = int( float( objUp[ 1 ] ) * 1000 )
-                if timestampBeginPtUp < 1400000000000:
-                    timestampBeginPtUp = \
-                        int( float( objUp[ 2 ] ) * 1000 )
-                main.log.info( "Port up begin timestamp: " +
-                               str( timestampBeginPtUp ) )
-            else:
-                main.log.info( "Tshark output file returned unexpected" +
-                               " results." )
-                timestampBeginPtUp = 0
-            fPortUp.close()
-
-            # Obtain metrics shortly afterwards
-            # This timestsamp captures
-            # the epoch time at which the topology graph was updated.
-            main.step( "Obtain t1 by REST call" )
-            jsonStrUp1 = main.ONOS1cli.topologyEventsMetrics()
-            jsonObj1 = json.loads( jsonStrUp1 )
-            graphTimestamp1 = \
-                jsonObj1[ graphTimestamp ][ 'value' ]
-            # Obtain device timestamp. This timestamp captures
-            # the epoch time at which the device event happened
-            deviceTimestamp1 = \
-                jsonObj1[ deviceTimestamp ][ 'value' ]
-            # Get delta between graph event and OFP
-            ptUpGraphToOfp1 = int( graphTimestamp1 ) -\
-                int( timestampBeginPtUp )
-            # Get delta between device event and OFP
-            ptUpDeviceToOfp1 = int( deviceTimestamp1 ) -\
-                int( timestampBeginPtUp )
-            
-            if ptUpGraphToOfp1 > upThresholdMin and\
-               ptUpGraphToOfp1 < upThresholdMax and i > iterIgnore:
-                portUpGraphNodeIter[0][i] = ptUpGraphToOfp1
-                main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
-                              str(ptUpGraphToOfp1)+" ms")
-            else:
-                main.log.info("iter"+str(i)+" skipped. Result: "+
-                              str(ptUpGraphToOfp1)+" ms")
-            
-            if ptUpDeviceToOfp1 > upThresholdMin and\
-               ptUpDeviceToOfp1 < upThresholdMax and i > iterIgnore:
-                portUpDevNodeIter[0][i] = ptUpDeviceToOfp1
-                main.log.info("iter"+str(i)+" port up device-to-ofp: "+
-                              str(ptUpDeviceToOfp1)+" ms")
-            else:
-                main.log.info("iter"+str(i)+" skipped. Result: "+
-                              str(ptUpDeviceToOfp1)+" ms")
-
-            if clusterCount >= 3:
-                jsonStrUp2 = main.ONOS2cli.topologyEventsMetrics()
-                jsonStrUp3 = main.ONOS3cli.topologyEventsMetrics()
-                jsonObj2 = json.loads( jsonStrUp2 )
-                jsonObj3 = json.loads( jsonStrUp3 )
-                graphTimestamp2 = \
-                    jsonObj2[ graphTimestamp ][ 'value' ]
-                graphTimestamp3 = \
-                    jsonObj3[ graphTimestamp ][ 'value' ]
-                deviceTimestamp2 = \
-                    jsonObj2[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp3 = \
-                    jsonObj3[ deviceTimestamp ][ 'value' ]
-                ptUpGraphToOfp2 = int( graphTimestamp2 ) -\
-                    int( timestampBeginPtUp )
-                ptUpGraphToOfp3 = int( graphTimestamp3 ) -\
-                    int( timestampBeginPtUp )
-                ptUpDeviceToOfp2 = int( deviceTimestamp2 ) -\
-                    int( timestampBeginPtUp )
-                ptUpDeviceToOfp3 = int( deviceTimestamp3 ) -\
-                    int( timestampBeginPtUp )
-            
-                if ptUpGraphToOfp2 > upThresholdMin and\
-                   ptUpGraphToOfp2 < upThresholdMax and i > iterIgnore:
-                    portUpGraphNodeIter[1][i] = ptUpGraphToOfp2
-                    main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
-                              str(ptUpGraphToOfp2)+" ms")
-            
-                if ptUpDeviceToOfp2 > upThresholdMin and\
-                   ptUpDeviceToOfp2 < upThresholdMax and i > iterIgnore:
-                    portUpDevNodeIter[1][i] = ptUpDeviceToOfp2
-                    main.log.info("iter"+str(i)+" port up device-to-ofp: "+
-                              str(ptUpDeviceToOfp2)+" ms")
-                
-                if ptUpGraphToOfp3 > upThresholdMin and\
-                   ptUpGraphToOfp3 < upThresholdMax and i > iterIgnore:
-                    portUpGraphNodeIter[2][i] = ptUpGraphToOfp3
-                    main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
-                              str(ptUpGraphToOfp3)+" ms")
-            
-                if ptUpDeviceToOfp3 > upThresholdMin and\
-                   ptUpDeviceToOfp3 < upThresholdMax and i > iterIgnore:
-                    portUpDevNodeIter[2][i] = ptUpDeviceToOfp3
-                    main.log.info("iter"+str(i)+" port up device-to-ofp: "+
-                              str(ptUpDeviceToOfp3)+" ms")
-
-            if clusterCount >= 5:
-                jsonStrUp4 = main.ONOS4cli.topologyEventsMetrics()
-                jsonStrUp5 = main.ONOS5cli.topologyEventsMetrics()
-                jsonObj4 = json.loads( jsonStrUp4 )
-                jsonObj5 = json.loads( jsonStrUp5 )
-                graphTimestamp4 = \
-                    jsonObj4[ graphTimestamp ][ 'value' ]
-                graphTimestamp5 = \
-                    jsonObj5[ graphTimestamp ][ 'value' ]
-                deviceTimestamp4 = \
-                    jsonObj4[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp5 = \
-                    jsonObj5[ deviceTimestamp ][ 'value' ]
-                ptUpGraphToOfp4 = int( graphTimestamp4 ) -\
-                    int( timestampBeginPtUp )
-                ptUpGraphToOfp5 = int( graphTimestamp5 ) -\
-                    int( timestampBeginPtUp )
-                ptUpDeviceToOfp4 = int( deviceTimestamp4 ) -\
-                    int( timestampBeginPtUp )
-                ptUpDeviceToOfp5 = int( deviceTimestamp5 ) -\
-                    int( timestampBeginPtUp )
-                
-                if ptUpGraphToOfp4 > upThresholdMin and\
-                   ptUpGraphToOfp4 < upThresholdMax and i > iterIgnore:
-                    portUpGraphNodeIter[3][i] = ptUpGraphToOfp4
-                    main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
-                              str(ptUpGraphToOfp4)+" ms")
-            
-                if ptUpDeviceToOfp4 > upThresholdMin and\
-                   ptUpDeviceToOfp4 < upThresholdMax and i > iterIgnore:
-                    portUpDevNodeIter[3][i] = ptUpDeviceToOfp4
-                    main.log.info("iter"+str(i)+" port up device-to-ofp: "+
-                              str(ptUpDeviceToOfp4)+" ms")
-                
-                if ptUpGraphToOfp5 > upThresholdMin and\
-                   ptUpGraphToOfp5 < upThresholdMax and i > iterIgnore:
-                    portUpGraphNodeIter[4][i] = ptUpGraphToOfp5
-                    main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
-                              str(ptUpGraphToOfp5)+" ms")
-            
-                if ptUpDeviceToOfp5 > upThresholdMin and\
-                   ptUpDeviceToOfp5 < upThresholdMax and i > iterIgnore:
-                    portUpDevNodeIter[4][i] = ptUpDeviceToOfp5
-                    main.log.info("iter"+str(i)+" port up device-to-ofp: "+
-                              str(ptUpDeviceToOfp5)+" ms")
-
-            if clusterCount >= 7:
-                jsonStrUp6 = main.ONOS6cli.topologyEventsMetrics()
-                jsonStrUp7 = main.ONOS7cli.topologyEventsMetrics()
-                jsonObj6 = json.loads( jsonStrUp6 )
-                jsonObj7 = json.loads( jsonStrUp7 )
-                graphTimestamp6 = \
-                    jsonObj6[ graphTimestamp ][ 'value' ]
-                graphTimestamp7 = \
-                    jsonObj7[ graphTimestamp ][ 'value' ]
-                deviceTimestamp6 = \
-                    jsonObj6[ deviceTimestamp ][ 'value' ]
-                deviceTimestamp7 = \
-                    jsonObj7[ deviceTimestamp ][ 'value' ]
-                ptUpGraphToOfp6 = int( graphTimestamp6 ) -\
-                    int( timestampBeginPtUp )
-                ptUpGraphToOfp7 = int( graphTimestamp7 ) -\
-                    int( timestampBeginPtUp )
-                ptUpDeviceToOfp6 = int( deviceTimestamp6 ) -\
-                    int( timestampBeginPtUp )
-                ptUpDeviceToOfp7 = int( deviceTimestamp7 ) -\
-                    int( timestampBeginPtUp )
-                
-                if ptUpGraphToOfp6 > upThresholdMin and\
-                   ptUpGraphToOfp6 < upThresholdMax and i > iterIgnore:
-                    portUpGraphNodeIter[5][i] = ptUpGraphToOfp6
-                    main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
-                              str(ptUpGraphToOfp6)+" ms")
-            
-                if ptUpDeviceToOfp6 > upThresholdMin and\
-                   ptUpDeviceToOfp6 < upThresholdMax and i > iterIgnore:
-                    portUpDevNodeIter[5][i] = ptUpDeviceToOfp6
-                    main.log.info("iter"+str(i)+" port up device-to-ofp: "+
-                              str(ptUpDeviceToOfp6)+" ms")
-                
-                if ptUpGraphToOfp7 > upThresholdMin and\
-                   ptUpGraphToOfp7 < upThresholdMax and i > iterIgnore:
-                    portUpGraphNodeIter[6][i] = ptUpGraphToOfp7
-                    main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
-                              str(ptUpGraphToOfp7)+" ms")
-            
-                if ptUpDeviceToOfp7 > upThresholdMin and\
-                   ptUpDeviceToOfp7 < upThresholdMax and i > iterIgnore:
-                    portUpDevNodeIter[6][i] = ptUpDeviceToOfp7
-                    main.log.info("iter"+str(i)+" port up device-to-ofp: "+
-                              str(ptUpDeviceToOfp7)+" ms")
-
-            # END ITERATION FOR LOOP
-        
-        portUpDevList = []
-        portUpGraphList = []
-        portDownDevList = []
-        portDownGraphList = []
-
-        portUpDevAvg = 0
-        portUpGraphAvg = 0
-        portDownDevAvg = 0
-        portDownGraphAvg = 0
-
-        dbCmdList = []
-
-        for node in range( 0, clusterCount ):
-
-            # NOTE: 
-            # Currently the 2d array is initialized with 0's. 
-            # We want to avoid skewing our results if the array
-            # was not modified with the correct latency.
-            for item in portUpDevNodeIter[node]:
-                if item > 0.0:
-                    portUpDevList.append(item)
-            for item in portUpGraphNodeIter[node]:
-                if item > 0.0:
-                    portUpGraphList.append(item)
-            for item in portDownDevNodeIter[node]:
-                if item > 0.0:
-                    portDownDevList.append(item)
-            for item in portDownGraphNodeIter[node]:
-                if item > 0.0:
-                    portDownGraphList.append(item)
-       
-            portUpDevAvg = round(numpy.mean(portUpDevList), 2)
-            portUpGraphAvg = round(numpy.mean(portUpGraphList), 2)
-            portDownDevAvg = round(numpy.mean(portDownDevList), 2)
-            portDownGraphAvg = round(numpy.mean(portDownGraphList), 2)
-
-            portUpStdDev = round(numpy.std(portUpGraphList),2)
-            portDownStdDev = round(numpy.std(portDownGraphList),2)
-
-            main.log.report( " - Node "+str(node+1)+" Summary - " )
-            #main.log.report( " Port up ofp-to-device "+
-            #                 str(round(portUpDevAvg, 2))+" ms")
-            main.log.report( " Port up ofp-to-graph "+
-                             str(portUpGraphAvg)+" ms")
-            #main.log.report( " Port down ofp-to-device "+
-            #                 str(round(portDownDevAvg, 2))+" ms")
-            main.log.report( " Port down ofp-to-graph "+
-                             str(portDownGraphAvg)+" ms")
-       
-            dbCmdList.append(
-                "INSERT INTO port_latency_tests VALUES("
-                   "'"+timeToPost+"','port_latency_results',"
-                   ""+runNum+","+str(clusterCount)+",'baremetal"+str(node+1)+"',"
-                   ""+str(portUpGraphAvg)+","+str(portUpStdDev)+
-                   ","+str(portDownGraphAvg)+","+str(portDownStdDev)+");"
-            )
-        
-        #Write to file for posting to DB
-        fResult = open(resultPath, 'a')
-        for line in dbCmdList:
-            if line:
-                fResult.write(line+"\n")
-        fResult.close()
-
-        print dbCmdList
-
-        # Remove switches from controller for next test
-        main.Mininet1.deleteSwController( "s1" )
-        main.Mininet1.deleteSwController( "s2" )
-
-        #TODO: correct assertion
-
-        utilities.assert_equals(
-            expect=main.TRUE,
-            actual=assertion,
-            onpass="Port discovery latency calculation successful",
-            onfail="Port discovery test failed" )
-
-    def CASE4( self, main ):
-        """
-        Link down event using loss rate 100%
-
-        Important:
-            Use a simple 2 switch topology with 1 link between
-            the two switches. Ensure that mac addresses of the
-            switches are 1 / 2 respectively
-        """
-        import time
-        import subprocess
-        import os
-        import requests
-        import json
-        import numpy
-
-        ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
-        ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
-        ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
-        ONOSUser = main.params[ 'CTRL' ][ 'user' ]
-
-        defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
-
-        # Number of iterations of case
-        numIter = main.params[ 'TEST' ][ 'numIter' ]
-
-        # Timestamp 'keys' for json metrics output.
-        # These are subject to change, hence moved into params
-        deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
-        linkTimestamp = main.params[ 'JSON' ][ 'linkTimestamp' ]
-        graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
-
-        debugMode = main.params[ 'TEST' ][ 'debugMode' ]
-
-        localTime = time.strftime( '%x %X' )
-        localTime = localTime.replace( "/", "" )
-        localTime = localTime.replace( " ", "_" )
-        localTime = localTime.replace( ":", "" )
-        if debugMode == 'on':
-            main.ONOS1.tsharkPcap( "eth0",
-                                   "/tmp/link_lat_pcap_" + localTime )
-
-        # Threshold for this test case
-        upThresholdStr = main.params[ 'TEST' ][ 'linkUpThreshold' ]
-        downThresholdStr = main.params[ 'TEST' ][ 'linkDownThreshold' ]
-
-        upThresholdObj = upThresholdStr.split( "," )
-        downThresholdObj = downThresholdStr.split( "," )
-
-        upThresholdMin = int( upThresholdObj[ 0 ] )
-        upThresholdMax = int( upThresholdObj[ 1 ] )
-
-        downThresholdMin = int( downThresholdObj[ 0 ] )
-        downThresholdMax = int( downThresholdObj[ 1 ] )
-
-        assertion = main.TRUE
-        # Link event timestamp to system time list
-        linkDownLinkToSystemList = []
-        linkUpLinkToSystemList = []
-        # Graph event timestamp to system time list
-        linkDownGraphToSystemList = []
-        linkUpGraphToSystemList = []
-
-        main.log.report( "Link up / down discovery latency between " +
-                         "two switches" )
-        main.log.report( "Simulated by setting loss-rate 100%" )
-        main.log.report( "'tc qdisc add dev <intfs> root netem loss 100%'" )
-        main.log.report( "Total iterations of test: " + str( numIter ) )
-
-        main.step( "Assign all switches" )
-        main.Mininet1.assignSwController( sw="1",
-                                           ip1=ONOS1Ip, port1=defaultSwPort )
-        main.Mininet1.assignSwController( sw="2",
-                                           ip1=ONOS1Ip, port1=defaultSwPort )
-
-        main.step( "Verifying switch assignment" )
-        resultS1 = main.Mininet1.getSwController( sw="s1" )
-        resultS2 = main.Mininet1.getSwController( sw="s2" )
-
-        # Allow time for events to finish before taking measurements
-        time.sleep( 10 )
-
-        linkDown1 = False
-        linkDown2 = False
-        linkDown3 = False
-        # Start iteration of link event test
-        for i in range( 0, int( numIter ) ):
-            main.step( "Getting initial system time as t0" )
-
-            # System time in epoch ms
-            timestampLinkDownT0 = time.time() * 1000
-            # Link down is simulated by 100% loss rate using traffic
-            # control command
-            main.Mininet1.handle.sendline(
-                "sh tc qdisc add dev s1-eth1 root netem loss 100%" )
-
-            # TODO: Iterate through 'links' command to verify that
-            #      link s1 -> s2 went down ( loop timeout 30 seconds )
-            #      on all 3 ONOS instances
-            main.log.info( "Checking ONOS for link update" )
-            loopCount = 0
-            while( not ( linkDown1 and linkDown2 and linkDown3 )
-                    and loopCount < 30 ):
-                jsonStr1 = main.ONOS1cli.links()
-                jsonStr2 = main.ONOS2cli.links()
-                jsonStr3 = main.ONOS3cli.links()
-
-                if not ( jsonStr1 and jsonStr2 and jsonStr3 ):
-                    main.log.error( "CLI command returned error " )
-                    break
-                else:
-                    jsonObj1 = json.loads( jsonStr1 )
-                    jsonObj2 = json.loads( jsonStr2 )
-                    jsonObj3 = json.loads( jsonStr3 )
-                for obj1 in jsonObj1:
-                    if '01' not in obj1[ 'src' ][ 'device' ]:
-                        linkDown1 = True
-                        main.log.info( "Link down from " +
-                                       "s1 -> s2 on ONOS1 detected" )
-                for obj2 in jsonObj2:
-                    if '01' not in obj2[ 'src' ][ 'device' ]:
-                        linkDown2 = True
-                        main.log.info( "Link down from " +
-                                       "s1 -> s2 on ONOS2 detected" )
-                for obj3 in jsonObj3:
-                    if '01' not in obj3[ 'src' ][ 'device' ]:
-                        linkDown3 = True
-                        main.log.info( "Link down from " +
-                                       "s1 -> s2 on ONOS3 detected" )
-
-                loopCount += 1
-                # If CLI doesn't like the continuous requests
-                # and exits in this loop, increase the sleep here.
-                # Consequently, while loop timeout will increase
-                time.sleep( 1 )
-
-            # Give time for metrics measurement to catch up
-            # NOTE: May need to be configured more accurately
-            time.sleep( 10 )
-            # If we exited the while loop and link down 1,2,3 are still
-            # false, then ONOS has failed to discover link down event
-            if not ( linkDown1 and linkDown2 and linkDown3 ):
-                main.log.info( "Link down discovery failed" )
-
-                linkDownLatGraph1 = 0
-                linkDownLatGraph2 = 0
-                linkDownLatGraph3 = 0
-                linkDownLatDevice1 = 0
-                linkDownLatDevice2 = 0
-                linkDownLatDevice3 = 0
-
-                assertion = main.FALSE
-            else:
-                jsonTopoMetrics1 =\
-                    main.ONOS1cli.topologyEventsMetrics()
-                jsonTopoMetrics2 =\
-                    main.ONOS2cli.topologyEventsMetrics()
-                jsonTopoMetrics3 =\
-                    main.ONOS3cli.topologyEventsMetrics()
-                jsonTopoMetrics1 = json.loads( jsonTopoMetrics1 )
-                jsonTopoMetrics2 = json.loads( jsonTopoMetrics2 )
-                jsonTopoMetrics3 = json.loads( jsonTopoMetrics3 )
-
-                main.log.info( "Obtaining graph and device timestamp" )
-                graphTimestamp1 = \
-                    jsonTopoMetrics1[ graphTimestamp ][ 'value' ]
-                graphTimestamp2 = \
-                    jsonTopoMetrics2[ graphTimestamp ][ 'value' ]
-                graphTimestamp3 = \
-                    jsonTopoMetrics3[ graphTimestamp ][ 'value' ]
-
-                linkTimestamp1 = \
-                    jsonTopoMetrics1[ linkTimestamp ][ 'value' ]
-                linkTimestamp2 = \
-                    jsonTopoMetrics2[ linkTimestamp ][ 'value' ]
-                linkTimestamp3 = \
-                    jsonTopoMetrics3[ linkTimestamp ][ 'value' ]
-
-                if graphTimestamp1 and graphTimestamp2 and\
-                        graphTimestamp3 and linkTimestamp1 and\
-                        linkTimestamp2 and linkTimestamp3:
-                    linkDownLatGraph1 = int( graphTimestamp1 ) -\
-                        int( timestampLinkDownT0 )
-                    linkDownLatGraph2 = int( graphTimestamp2 ) -\
-                        int( timestampLinkDownT0 )
-                    linkDownLatGraph3 = int( graphTimestamp3 ) -\
-                        int( timestampLinkDownT0 )
-
-                    linkDownLatLink1 = int( linkTimestamp1 ) -\
-                        int( timestampLinkDownT0 )
-                    linkDownLatLink2 = int( linkTimestamp2 ) -\
-                        int( timestampLinkDownT0 )
-                    linkDownLatLink3 = int( linkTimestamp3 ) -\
-                        int( timestampLinkDownT0 )
-                else:
-                    main.log.error( "There was an error calculating" +
-                                    " the delta for link down event" )
-                    linkDownLatGraph1 = 0
-                    linkDownLatGraph2 = 0
-                    linkDownLatGraph3 = 0
-
-                    linkDownLatDevice1 = 0
-                    linkDownLatDevice2 = 0
-                    linkDownLatDevice3 = 0
-
-            main.log.info( "Link down latency ONOS1 iteration " +
-                           str( i ) + " (end-to-end): " +
-                           str( linkDownLatGraph1 ) + " ms" )
-            main.log.info( "Link down latency ONOS2 iteration " +
-                           str( i ) + " (end-to-end): " +
-                           str( linkDownLatGraph2 ) + " ms" )
-            main.log.info( "Link down latency ONOS3 iteration " +
-                           str( i ) + " (end-to-end): " +
-                           str( linkDownLatGraph3 ) + " ms" )
-
-            main.log.info( "Link down latency ONOS1 iteration " +
-                           str( i ) + " (link-event-to-system-timestamp): " +
-                           str( linkDownLatLink1 ) + " ms" )
-            main.log.info( "Link down latency ONOS2 iteration " +
-                           str( i ) + " (link-event-to-system-timestamp): " +
-                           str( linkDownLatLink2 ) + " ms" )
-            main.log.info( "Link down latency ONOS3 iteration " +
-                           str( i ) + " (link-event-to-system-timestamp): " +
-                           str( linkDownLatLink3 ) )
-
-            # Calculate avg of node calculations
-            linkDownLatGraphAvg =\
-                ( linkDownLatGraph1 +
-                  linkDownLatGraph2 +
-                  linkDownLatGraph3 ) / 3
-            linkDownLatLinkAvg =\
-                ( linkDownLatLink1 +
-                  linkDownLatLink2 +
-                  linkDownLatLink3 ) / 3
-
-            # Set threshold and append latency to list
-            if linkDownLatGraphAvg > downThresholdMin and\
-               linkDownLatGraphAvg < downThresholdMax:
-                linkDownGraphToSystemList.append(
-                    linkDownLatGraphAvg )
-            else:
-                main.log.info( "Link down latency exceeded threshold" )
-                main.log.info( "Results for iteration " + str( i ) +
-                               "have been omitted" )
-            if linkDownLatLinkAvg > downThresholdMin and\
-               linkDownLatLinkAvg < downThresholdMax:
-                linkDownLinkToSystemList.append(
-                    linkDownLatLinkAvg )
-            else:
-                main.log.info( "Link down latency exceeded threshold" )
-                main.log.info( "Results for iteration " + str( i ) +
-                               "have been omitted" )
-
-            # NOTE: To remove loss rate and measure latency:
-            #       'sh tc qdisc del dev s1-eth1 root'
-            timestampLinkUpT0 = time.time() * 1000
-            main.Mininet1.handle.sendline( "sh tc qdisc del dev " +
-                                           "s1-eth1 root" )
-            main.Mininet1.handle.expect( "mininet>" )
-
-            main.log.info( "Checking ONOS for link update" )
-
-            linkDown1 = True
-            linkDown2 = True
-            linkDown3 = True
-            loopCount = 0
-            while( ( linkDown1 and linkDown2 and linkDown3 )
-                    and loopCount < 30 ):
-                jsonStr1 = main.ONOS1cli.links()
-                jsonStr2 = main.ONOS2cli.links()
-                jsonStr3 = main.ONOS3cli.links()
-                if not ( jsonStr1 and jsonStr2 and jsonStr3 ):
-                    main.log.error( "CLI command returned error " )
-                    break
-                else:
-                    jsonObj1 = json.loads( jsonStr1 )
-                    jsonObj2 = json.loads( jsonStr2 )
-                    jsonObj3 = json.loads( jsonStr3 )
-
-                for obj1 in jsonObj1:
-                    if '01' in obj1[ 'src' ][ 'device' ]:
-                        linkDown1 = False
-                        main.log.info( "Link up from " +
-                                       "s1 -> s2 on ONOS1 detected" )
-                for obj2 in jsonObj2:
-                    if '01' in obj2[ 'src' ][ 'device' ]:
-                        linkDown2 = False
-                        main.log.info( "Link up from " +
-                                       "s1 -> s2 on ONOS2 detected" )
-                for obj3 in jsonObj3:
-                    if '01' in obj3[ 'src' ][ 'device' ]:
-                        linkDown3 = False
-                        main.log.info( "Link up from " +
-                                       "s1 -> s2 on ONOS3 detected" )
-
-                loopCount += 1
-                time.sleep( 1 )
-
-            if ( linkDown1 and linkDown2 and linkDown3 ):
-                main.log.info( "Link up discovery failed" )
-
-                linkUpLatGraph1 = 0
-                linkUpLatGraph2 = 0
-                linkUpLatGraph3 = 0
-                linkUpLatDevice1 = 0
-                linkUpLatDevice2 = 0
-                linkUpLatDevice3 = 0
-
-                assertion = main.FALSE
-            else:
-                jsonTopoMetrics1 =\
-                    main.ONOS1cli.topologyEventsMetrics()
-                jsonTopoMetrics2 =\
-                    main.ONOS2cli.topologyEventsMetrics()
-                jsonTopoMetrics3 =\
-                    main.ONOS3cli.topologyEventsMetrics()
-                jsonTopoMetrics1 = json.loads( jsonTopoMetrics1 )
-                jsonTopoMetrics2 = json.loads( jsonTopoMetrics2 )
-                jsonTopoMetrics3 = json.loads( jsonTopoMetrics3 )
-
-                main.log.info( "Obtaining graph and device timestamp" )
-                graphTimestamp1 = \
-                    jsonTopoMetrics1[ graphTimestamp ][ 'value' ]
-                graphTimestamp2 = \
-                    jsonTopoMetrics2[ graphTimestamp ][ 'value' ]
-                graphTimestamp3 = \
-                    jsonTopoMetrics3[ graphTimestamp ][ 'value' ]
-
-                linkTimestamp1 = \
-                    jsonTopoMetrics1[ linkTimestamp ][ 'value' ]
-                linkTimestamp2 = \
-                    jsonTopoMetrics2[ linkTimestamp ][ 'value' ]
-                linkTimestamp3 = \
-                    jsonTopoMetrics3[ linkTimestamp ][ 'value' ]
-
-                if graphTimestamp1 and graphTimestamp2 and\
-                        graphTimestamp3 and linkTimestamp1 and\
-                        linkTimestamp2 and linkTimestamp3:
-                    linkUpLatGraph1 = int( graphTimestamp1 ) -\
-                        int( timestampLinkUpT0 )
-                    linkUpLatGraph2 = int( graphTimestamp2 ) -\
-                        int( timestampLinkUpT0 )
-                    linkUpLatGraph3 = int( graphTimestamp3 ) -\
-                        int( timestampLinkUpT0 )
-
-                    linkUpLatLink1 = int( linkTimestamp1 ) -\
-                        int( timestampLinkUpT0 )
-                    linkUpLatLink2 = int( linkTimestamp2 ) -\
-                        int( timestampLinkUpT0 )
-                    linkUpLatLink3 = int( linkTimestamp3 ) -\
-                        int( timestampLinkUpT0 )
-                else:
-                    main.log.error( "There was an error calculating" +
-                                    " the delta for link down event" )
-                    linkUpLatGraph1 = 0
-                    linkUpLatGraph2 = 0
-                    linkUpLatGraph3 = 0
-
-                    linkUpLatDevice1 = 0
-                    linkUpLatDevice2 = 0
-                    linkUpLatDevice3 = 0
-
-            if debugMode == 'on':
-                main.log.info( "Link up latency ONOS1 iteration " +
-                               str( i ) + " (end-to-end): " +
-                               str( linkUpLatGraph1 ) + " ms" )
-                main.log.info( "Link up latency ONOS2 iteration " +
-                               str( i ) + " (end-to-end): " +
-                               str( linkUpLatGraph2 ) + " ms" )
-                main.log.info( "Link up latency ONOS3 iteration " +
-                               str( i ) + " (end-to-end): " +
-                               str( linkUpLatGraph3 ) + " ms" )
-
-                main.log.info(
-                    "Link up latency ONOS1 iteration " +
-                    str( i ) +
-                    " (link-event-to-system-timestamp): " +
-                    str( linkUpLatLink1 ) +
-                    " ms" )
-                main.log.info(
-                    "Link up latency ONOS2 iteration " +
-                    str( i ) +
-                    " (link-event-to-system-timestamp): " +
-                    str( linkUpLatLink2 ) +
-                    " ms" )
-                main.log.info(
-                    "Link up latency ONOS3 iteration " +
-                    str( i ) +
-                    " (link-event-to-system-timestamp): " +
-                    str( linkUpLatLink3 ) )
-
-            # Calculate avg of node calculations
-            linkUpLatGraphAvg =\
-                ( linkUpLatGraph1 +
-                  linkUpLatGraph2 +
-                  linkUpLatGraph3 ) / 3
-            linkUpLatLinkAvg =\
-                ( linkUpLatLink1 +
-                  linkUpLatLink2 +
-                  linkUpLatLink3 ) / 3
-
-            # Set threshold and append latency to list
-            if linkUpLatGraphAvg > upThresholdMin and\
-               linkUpLatGraphAvg < upThresholdMax:
-                linkUpGraphToSystemList.append(
-                    linkUpLatGraphAvg )
-            else:
-                main.log.info( "Link up latency exceeded threshold" )
-                main.log.info( "Results for iteration " + str( i ) +
-                               "have been omitted" )
-            if linkUpLatLinkAvg > upThresholdMin and\
-               linkUpLatLinkAvg < upThresholdMax:
-                linkUpLinkToSystemList.append(
-                    linkUpLatLinkAvg )
-            else:
-                main.log.info( "Link up latency exceeded threshold" )
-                main.log.info( "Results for iteration " + str( i ) +
-                               "have been omitted" )
-
-        # Calculate min, max, avg of list and report
-        linkDownMin = min( linkDownGraphToSystemList )
-        linkDownMax = max( linkDownGraphToSystemList )
-        linkDownAvg = sum( linkDownGraphToSystemList ) / \
-            len( linkDownGraphToSystemList )
-        linkUpMin = min( linkUpGraphToSystemList )
-        linkUpMax = max( linkUpGraphToSystemList )
-        linkUpAvg = sum( linkUpGraphToSystemList ) / \
-            len( linkUpGraphToSystemList )
-        linkDownStdDev = \
-            str( round( numpy.std( linkDownGraphToSystemList ), 1 ) )
-        linkUpStdDev = \
-            str( round( numpy.std( linkUpGraphToSystemList ), 1 ) )
-
-        main.log.report( "Link down latency " +
-                         "Avg: " + str( linkDownAvg ) + " ms " +
-                         "Std Deviation: " + linkDownStdDev + " ms" )
-        main.log.report( "Link up latency " +
-                         "Avg: " + str( linkUpAvg ) + " ms " +
-                         "Std Deviation: " + linkUpStdDev + " ms" )
-
-        utilities.assert_equals(
-            expect=main.TRUE,
-            actual=assertion,
-            onpass="Link discovery latency calculation successful",
-            onfail="Link discovery latency case failed" )
-
-    def CASE5( self, main ):
-        """
-        100 Switch discovery latency
-
-        Important:
-            This test case can be potentially dangerous if
-            your machine has previously set iptables rules.
-            One of the steps of the test case will flush
-            all existing iptables rules.
-        Note:
-            You can specify the number of switches in the
-            params file to adjust the switch discovery size
-            ( and specify the corresponding topology in Mininet1
-            .topo file )
-        """
-        import time
-        import subprocess
-        import os
-        import requests
-        import json
-
-        ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
-        ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
-        ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
-        MN1Ip = main.params[ 'MN' ][ 'ip1' ]
-        ONOSUser = main.params[ 'CTRL' ][ 'user' ]
-
-        defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
-
-        # Number of iterations of case
-        numIter = main.params[ 'TEST' ][ 'numIter' ]
-        numSw = main.params[ 'TEST' ][ 'numSwitch' ]
-
-        # Timestamp 'keys' for json metrics output.
-        # These are subject to change, hence moved into params
-        deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
-        graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
-
-        debugMode = main.params[ 'TEST' ][ 'debugMode' ]
-
-        localTime = time.strftime( '%X' )
-        localTime = localTime.replace( "/", "" )
-        localTime = localTime.replace( " ", "_" )
-        localTime = localTime.replace( ":", "" )
-        if debugMode == 'on':
-            main.ONOS1.tsharkPcap( "eth0",
-                                   "/tmp/100_sw_lat_pcap_" + localTime )
-
-        # Threshold for this test case
-        swDiscThresholdStr = main.params[ 'TEST' ][ 'swDisc100Threshold' ]
-        swDiscThresholdObj = swDiscThresholdStr.split( "," )
-        swDiscThresholdMin = int( swDiscThresholdObj[ 0 ] )
-        swDiscThresholdMax = int( swDiscThresholdObj[ 1 ] )
-
-        tsharkOfpOutput = "/tmp/tshark_ofp_" + numSw + "sw.txt"
-        tsharkTcpOutput = "/tmp/tshark_tcp_" + numSw + "sw.txt"
-
-        tsharkOfpResultList = []
-        tsharkTcpResultList = []
-
-        swDiscoveryLatList = []
-
-        main.case( numSw + " Switch discovery latency" )
-        main.step( "Assigning all switches to ONOS1" )
-        for i in range( 1, int( numSw ) + 1 ):
-            main.Mininet1.assignSwController(
-                sw=str( i ),
-                ip1=ONOS1Ip,
-                port1=defaultSwPort )
-
-        # Ensure that nodes are configured with ptpd
-        # Just a warning message
-        main.log.info( "Please check ptpd configuration to ensure" +
-                       " All nodes' system times are in sync" )
-        time.sleep( 5 )
-
-        for i in range( 0, int( numIter ) ):
-
-            main.step( "Set iptables rule to block incoming sw connections" )
-            # Set iptables rule to block incoming switch connections
-            # The rule description is as follows:
-            #   Append to INPUT rule,
-            #   behavior DROP that matches following:
-            #       * packet type: tcp
-            #       * source IP: MN1Ip
-            #       * destination PORT: 6633
-            main.ONOS1.handle.sendline(
-                "sudo iptables -A INPUT -p tcp -s " + MN1Ip +
-                " --dport " + defaultSwPort + " -j DROP" )
-            main.ONOS1.handle.expect( "\$" )
-            #   Append to OUTPUT rule,
-            #   behavior DROP that matches following:
-            #       * packet type: tcp
-            #       * source IP: MN1Ip
-            #       * destination PORT: 6633
-            main.ONOS1.handle.sendline(
-                "sudo iptables -A OUTPUT -p tcp -s " + MN1Ip +
-                " --dport " + defaultSwPort + " -j DROP" )
-            main.ONOS1.handle.expect( "\$" )
-            # Give time to allow rule to take effect
-            # NOTE: Sleep period may need to be configured
-            #      based on the number of switches in the topology
-            main.log.info( "Please wait for switch connection to " +
-                           "time out" )
-            time.sleep( 60 )
-
-            # Gather vendor OFP with tshark
-            main.ONOS1.tsharkGrep( "OFP 86 Vendor",
-                                   tsharkOfpOutput )
-            main.ONOS1.tsharkGrep( "TCP 74 ",
-                                   tsharkTcpOutput )
-
-            # NOTE: Remove all iptables rule quickly ( flush )
-            #      Before removal, obtain TestON timestamp at which
-            #      removal took place
-            #      ( ensuring nodes are configured via ptp )
-            #      sudo iptables -F
-
-            t0System = time.time() * 1000
-            main.ONOS1.handle.sendline(
-                "sudo iptables -F" )
-
-            # Counter to track loop count
-            counterLoop = 0
-            counterAvail1 = 0
-            counterAvail2 = 0
-            counterAvail3 = 0
-            onos1Dev = False
-            onos2Dev = False
-            onos3Dev = False
-            while counterLoop < 60:
-                # Continue to check devices for all device
-                # availability. When all devices in all 3
-                # ONOS instances indicate that devices are available
-                # obtain graph event timestamp for t1.
-                deviceStrObj1 = main.ONOS1cli.devices()
-                deviceStrObj2 = main.ONOS2cli.devices()
-                deviceStrObj3 = main.ONOS3cli.devices()
-
-                deviceJson1 = json.loads( deviceStrObj1 )
-                deviceJson2 = json.loads( deviceStrObj2 )
-                deviceJson3 = json.loads( deviceStrObj3 )
-
-                for device1 in deviceJson1:
-                    if device1[ 'available' ]:
-                        counterAvail1 += 1
-                        if counterAvail1 == int( numSw ):
-                            onos1Dev = True
-                            main.log.info( "All devices have been " +
-                                           "discovered on ONOS1" )
-                    else:
-                        counterAvail1 = 0
-                for device2 in deviceJson2:
-                    if device2[ 'available' ]:
-                        counterAvail2 += 1
-                        if counterAvail2 == int( numSw ):
-                            onos2Dev = True
-                            main.log.info( "All devices have been " +
-                                           "discovered on ONOS2" )
-                    else:
-                        counterAvail2 = 0
-                for device3 in deviceJson3:
-                    if device3[ 'available' ]:
-                        counterAvail3 += 1
-                        if counterAvail3 == int( numSw ):
-                            onos3Dev = True
-                            main.log.info( "All devices have been " +
-                                           "discovered on ONOS3" )
-                    else:
-                        counterAvail3 = 0
-
-                if onos1Dev and onos2Dev and onos3Dev:
-                    main.log.info( "All devices have been discovered " +
-                                   "on all ONOS instances" )
-                    jsonStrTopologyMetrics1 =\
-                        main.ONOS1cli.topologyEventsMetrics()
-                    jsonStrTopologyMetrics2 =\
-                        main.ONOS2cli.topologyEventsMetrics()
-                    jsonStrTopologyMetrics3 =\
-                        main.ONOS3cli.topologyEventsMetrics()
-
-                    # Exit while loop if all devices discovered
-                    break
-
-                counterLoop += 1
-                # Give some time in between CLI calls
-                #( will not affect measurement )
-                time.sleep( 3 )
-
-            main.ONOS1.tsharkStop()
-
-            os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
-                       tsharkOfpOutput + " /tmp/" )
-            os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
-                       tsharkTcpOutput + " /tmp/" )
-
-            # TODO: Automate OFP output analysis
-            # Debug mode - print out packets captured at runtime
-            if debugMode == 'on':
-                ofpFile = open( tsharkOfpOutput, 'r' )
-                main.log.info( "Tshark OFP Vendor output: " )
-                for line in ofpFile:
-                    tsharkOfpResultList.append( line )
-                    main.log.info( line )
-                ofpFile.close()
-
-                tcpFile = open( tsharkTcpOutput, 'r' )
-                main.log.info( "Tshark TCP 74 output: " )
-                for line in tcpFile:
-                    tsharkTcpResultList.append( line )
-                    main.log.info( line )
-                tcpFile.close()
-
-            jsonObj1 = json.loads( jsonStrTopologyMetrics1 )
-            jsonObj2 = json.loads( jsonStrTopologyMetrics2 )
-            jsonObj3 = json.loads( jsonStrTopologyMetrics3 )
-
-            graphTimestamp1 = \
-                jsonObj1[ graphTimestamp ][ 'value' ]
-            graphTimestamp2 = \
-                jsonObj2[ graphTimestamp ][ 'value' ]
-            graphTimestamp3 = \
-                jsonObj3[ graphTimestamp ][ 'value' ]
-
-            graphLat1 = int( graphTimestamp1 ) - int( t0System )
-            graphLat2 = int( graphTimestamp2 ) - int( t0System )
-            graphLat3 = int( graphTimestamp3 ) - int( t0System )
-
-            avgGraphLat = \
-                ( int( graphLat1 ) +
-                  int( graphLat2 ) +
-                  int( graphLat3 ) ) / 3
-
-            if avgGraphLat > swDiscThresholdMin \
-                    and avgGraphLat < swDiscThresholdMax:
-                swDiscoveryLatList.append(
-                    avgGraphLat )
-            else:
-                main.log.info( "100 Switch discovery latency " +
-                               "exceeded the threshold." )
-
-            # END ITERATION FOR LOOP
-
-        swLatMin = min( swDiscoveryLatList )
-        swLatMax = max( swDiscoveryLatList )
-        swLatAvg = sum( swDiscoveryLatList ) /\
-            len( swDiscoveryLatList )
-
-        main.log.report( "100 Switch discovery lat " +
-                         "Min: " + str( swLatMin ) + " ms" +
-                         "Max: " + str( swLatMax ) + " ms" +
-                         "Avg: " + str( swLatAvg ) + " ms" )
-
-    def CASE6( self, main ):
-        """
-        Increase number of nodes and initiate CLI
-        """
-        import time
-
-        ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
-        ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
-        ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
-        ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
-        ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
-        ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
-        ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
-
-        cellName = main.params[ 'ENV' ][ 'cellName' ]
-
-        global clusterCount
-
-        # Cluster size increased everytime the case is defined
-        clusterCount += 2
-
-        main.log.report( "Increasing cluster size to " +
-                         str( clusterCount ) )
-
-        installResult = main.FALSE
-        if clusterCount == 3:
-            main.log.info( "Installing nodes 2 and 3" )
-            node2Result = \
-                main.ONOSbench.onosInstall( node=ONOS2Ip )
-            node3Result = \
-                main.ONOSbench.onosInstall( node=ONOS3Ip )
-            installResult = node2Result and node3Result
-
-            time.sleep( 5 )
-
-            main.ONOS2cli.startOnosCli( ONOS2Ip )
-            main.ONOS3cli.startOnosCli( ONOS3Ip )
-
-        elif clusterCount == 5:
-            main.log.info( "Installing nodes 4 and 5" )
-            node4Result = \
-                main.ONOSbench.onosInstall( node=ONOS4Ip )
-            node5Result = \
-                main.ONOSbench.onosInstall( node=ONOS5Ip )
-            installResult = node4Result and node5Result
-
-            time.sleep( 5 )
-
-            main.ONOS4cli.startOnosCli( ONOS4Ip )
-            main.ONOS5cli.startOnosCli( ONOS5Ip )
-
-        elif clusterCount == 7:
-            main.log.info( "Installing nodes 6 and 7" )
-            node6Result = \
-                main.ONOSbench.onosInstall( node=ONOS6Ip )
-            node7Result = \
-                main.ONOSbench.onosInstall( node=ONOS7Ip )
-            installResult = node6Result and node7Result
-
-            time.sleep( 5 )
-
-            main.ONOS6cli.startOnosCli( ONOS6Ip )
-            main.ONOS7cli.startOnosCli( ONOS7Ip )
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.topo b/TestON/tests/TopoPerfNext/TopoPerfNext.topo
deleted file mode 100644
index f12d192..0000000
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.topo
+++ /dev/null
@@ -1,163 +0,0 @@
-<TOPOLOGY>
-    <COMPONENT>
-        
-        <ONOSbench>
-            <host>10.128.174.10</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosDriver</type>
-            <connect_order>1</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOSbench>
-
-        <ONOS1cli>
-            <host>10.128.174.10</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosCliDriver</type>
-            <connect_order>2</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS1cli>
-
-        <ONOS2cli>
-            <host>10.128.174.10</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosCliDriver</type>
-            <connect_order>3</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS2cli>
-        
-        <ONOS3cli>
-            <host>10.128.174.10</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosCliDriver</type>
-            <connect_order>4</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS3cli>
-        
-        <ONOS4cli>
-            <host>10.128.174.10</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosCliDriver</type>
-            <connect_order>5</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS4cli>
-        
-        <ONOS5cli>
-            <host>10.128.174.10</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosCliDriver</type>
-            <connect_order>6</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS5cli>
-        
-        <ONOS6cli>
-            <host>10.128.174.10</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosCliDriver</type>
-            <connect_order>7</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS6cli>
-        
-        <ONOS7cli>
-            <host>10.128.174.10</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosCliDriver</type>
-            <connect_order>8</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS7cli>
-
-        <ONOS1>
-            <host>10.128.174.1</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosDriver</type>
-            <connect_order>9</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS1>
-
-        <ONOS2>
-            <host>10.128.174.2</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosDriver</type>
-            <connect_order>10</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS2>
-
-        <ONOS3>
-            <host>10.128.174.3</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosDriver</type>
-            <connect_order>11</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS3>
-        
-        <ONOS4>
-            <host>10.128.174.4</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosDriver</type>
-            <connect_order>12</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS4>
-        
-        <ONOS5>
-            <host>10.128.174.5</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosDriver</type>
-            <connect_order>13</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS5>
-        
-        <ONOS6>
-            <host>10.128.174.6</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosDriver</type>
-            <connect_order>14</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS6>
-        
-        <ONOS7>
-            <host>10.128.174.7</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>OnosDriver</type>
-            <connect_order>15</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </ONOS7>
-
-        <Mininet1>
-            <host>10.128.10.90</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>MininetCliDriver</type>
-            <connect_order>16</connect_order>
-            <COMPONENTS>
-                <arg1> --custom topo-perf-2sw.py </arg1>
-                <arg2> --arp --mac --topo mytopo</arg2>
-                <arg3> </arg3>
-                <controller> remote </controller>
-            </COMPONENTS>
-        </Mininet1>
-
-        <Mininet2>
-            <host>10.128.10.90</host>
-            <user>admin</user>
-            <password>onos_test</password>
-            <type>RemoteMininetDriver</type>
-            <connect_order>17</connect_order>
-            <COMPONENTS> </COMPONENTS>
-        </Mininet2>
-
-    </COMPONENT>
-</TOPOLOGY>
diff --git a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params
index ab215e9..eba319c 100644
--- a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params
+++ b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params
@@ -3,7 +3,7 @@
 
     <ENV>
         <cellName>topo_perf_test</cellName>
-        <cellFeatures>"org.onosproject.metrics,org.onosproject.openflow"</cellFeatures>
+        <cellFeatures>"drivers,metrics,openflow"</cellFeatures>
     </ENV>
 
     <GIT>
@@ -13,7 +13,7 @@
     </GIT>
 
     <CTRL>
-        <user>sdn</user>
+        <user>admin</user>
         <ip1>10.254.1.201</ip1>
         <port1>6633</port1>
         <ip2>10.254.1.202</ip2>
@@ -62,7 +62,7 @@
         </topoConfigName>
 
         #Number of times to iterate each case
-        <numIter>35</numIter>
+        <numIter>13</numIter>
         <numSwitch>2</numSwitch>
         #Number of iterations to ignore initially
         <iterIgnore>2</iterIgnore>
diff --git a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
index 276de24..e2245ba 100644
--- a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
+++ b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
@@ -39,9 +39,9 @@
         except KeyError:
             # Jenkins build number is also used in posting to DB
             # If this test is not triggered by jenkins, give 
-            # it the runNum variable instead, ensuring that 
+            # it 0 instead, ensuring that 
             # the DB post will recognize it as a non-jenkins run
-            jenkinsBuildNumber = str(runNum)
+            jenkinsBuildNumber = str(0)
             main.log.info( 'Job is not run by jenkins. '+
                     'Build number set to: ' + jenkinsBuildNumber)
 
@@ -192,8 +192,8 @@
         resultPath = main.params['DB']['switchEventResultPath']
         thresholdStr = main.params['TEST']['singleSwThreshold']
         thresholdObj = thresholdStr.split(',')
-        thresholdMin = float(thresholdObj[0])
-        thresholdMax = float(thresholdObj[1])
+        thresholdMin = int(thresholdObj[0])
+        thresholdMax = int(thresholdObj[1])
        
         # Look for 'role-request' messages,
         # which replaces the 'vendor' messages previously seen
@@ -266,11 +266,16 @@
             main.log.info('Assigning s3 to controller')
             main.Mininet1.assignSwController(sw='3',
                     ip1=nodeIpList[0], port1=defaultSwPort)
-            
-            time.sleep(10)
-            
+               
+            jsonStr = []
+            for node in range (0, clusterCount): 
+                metricsSwUp = CLIs[node].topologyEventsMetrics()
+                jsonStr.append(metricsSwUp)
+           
             main.log.info('Stopping all Tshark processes')
             main.ONOS1.tsharkStop()
+           
+            time.sleep(5)
             
             main.log.info('Copying over tshark files')
             os.system('scp ' + ONOSUser + '@' + nodeIpList[0] +
@@ -283,7 +288,8 @@
                       nodeIpList[0] + ':' + tsharkOfOutput + ' /tmp/')
            
             # Get tcp syn / ack output
-            time.sleep(5)
+            time.sleep(1)
+
             tcpFile = open(tsharkTcpOutput, 'r')
             tempText = tcpFile.readline()
             tempText = tempText.split(' ')
@@ -300,7 +306,6 @@
             tcpFile.close()
            
             # Get Role reply output
-            time.sleep(5)
             ofFile = open(tsharkOfOutput, 'r')
             lineOfp = ''
             while True:
@@ -343,7 +348,13 @@
             if len(tempText) > 1:
                 main.log.info('Object read in from feature reply capture: '+
                         str(tempText))
-                featureTimestamp = float(tempText[1]) * 1000.0
+                if tempText[1] != ' ' and float(tempText[1]) > 1400000000.0:
+                    temp = tempText[1]
+                elif tempText[2] != ' ' and float(tempText[2]) > 1400000000.0:
+                    temp = tempText[2]
+                else:
+                    temp = 0 
+                featureTimestamp = float(temp) * 1000.0
             else:
                 main.log.error('Tshark output file for feature reply' +
                         ' returned unexpected results')
@@ -351,16 +362,11 @@
                 assertion = main.FALSE
             featureFile.close()
 
-            # TODO: calculate feature reply, role request times
-            # stack measurements correctly and report
-
-            #TODO: Refactor in progress
-
             for node in range(0, clusterCount):
                 nodeNum = node+1
-                metricsSwUp = CLIs[node].topologyEventsMetrics
-                jsonStr = metricsSwUp()
-                jsonObj = json.loads(jsonStr)
+                #metricsSwUp = CLIs[node].topologyEventsMetrics
+                #jsonStr = metricsSwUp()
+                jsonObj = json.loads(jsonStr[node])
                 if jsonObj:
                     graphTimestamp = jsonObj[graphTimestampKey]['value']
                     deviceTimestamp = jsonObj[deviceTimestampKey]['value']
@@ -398,8 +404,9 @@
                 else:
                     main.log.info("ONOS "+str(nodeNum)+ " end-to-end "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
-
+                            "threshold or premature iteration: ")
+                    main.log.info(str(endToEnd))
+                        
                 if tcpToFeature >= thresholdMin and\
                    tcpToFeature < thresholdMax and i >= iterIgnore:
                     tcpToFeatureLatNodeIter[node][i] = tcpToFeature 
@@ -408,7 +415,8 @@
                 else:
                     main.log.info("ONOS "+str(nodeNum)+ " tcp-to-feature "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
+                            "threshold or premature iteration: ")
+                    main.log.info(str(tcpToFeature))
 
                 if featureToRole >= thresholdMin and\
                    featureToRole < thresholdMax and i >= iterIgnore:
@@ -418,7 +426,8 @@
                 else:
                     main.log.info("ONOS "+str(nodeNum)+ " feature-to-role "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
+                            "threshold or premature iteration: ")
+                    main.log.info(str(featureToRole))
 
                 if roleToOfp >= thresholdMin and\
                    roleToOfp < thresholdMax and i >= iterIgnore:
@@ -428,7 +437,8 @@
                 else:
                     main.log.info("ONOS "+str(nodeNum)+ " role-to-reply "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
+                            "threshold or premature iteration: ")
+                    main.log.info(str(roleToOfp))
                 
                 if ofpToDevice >= thresholdMin and\
                    ofpToDevice < thresholdMax and i >= iterIgnore:
@@ -438,7 +448,8 @@
                 else:
                     main.log.info("ONOS "+str(nodeNum)+ " reply-to-device "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
+                            "threshold or premature iteration: ")
+                    main.log.info(str(ofpToDevice))
 
                 if deviceToGraph >= thresholdMin and\
                    deviceToGraph < thresholdMax and i >= iterIgnore:
@@ -451,13 +462,14 @@
                         main.log.info("ONOS "+str(nodeNum) +
                             " device-to-graph measurement "+
                             "was set to 0 ms because of precision "+
-                            "uncertainty")
+                            "uncertainty. ")
                     else:
                         main.log.info("ONOS "+str(nodeNum)+ 
                             " device-to-graph "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
-                                
+                            "threshold or premature iteration: ")
+                        main.log.info(str(deviceToGraph))   
+                            
             # ********************
             time.sleep(5)
         
@@ -489,22 +501,21 @@
             main.step('Remove switch from controller')
             main.Mininet1.deleteSwController('s3')
             firstDevice = deviceList[0] 
-            main.log.info( "Removing device " +str(firstDevice)+
-                    " from ONOS" )
             
-            time.sleep( 5 )
-           
             # We need to get metrics before removing
             # device from the store below.
             for node in range(0, clusterCount):
                 metricsSwDown = CLIs[node].topologyEventsMetrics
                 jsonStr = metricsSwDown()
                 removeJsonList.append( json.loads(jsonStr) ) 
-            
-            #if deviceId:
-            main.ONOS1cli.deviceRemove(firstDevice)
            
             main.ONOS1.tsharkStop()
+            
+            main.log.info( "Removing device " +str(firstDevice)+
+                    " from ONOS" )
+           
+            #if deviceId:
+            main.ONOS1cli.deviceRemove(firstDevice)
 
             main.log.info('Copying over tshark files')
             os.system('scp ' + ONOSUser + '@' + nodeIpList[0] +
@@ -520,12 +531,28 @@
                 index = 1
                 for line in tempLine:
                     obj = line.split(' ')
-            
-                    if len(obj) > 1:
+           
+                    # There are at least 3 objects in field (valid 
+                    # tshark output is lengthy)
+                    if len(obj) > 2:
+                        # If first index of object is like an epoch time
+                        if obj[1] != ' ' and float(obj[1]) > 1400000000.0:
+                            temp = obj[1] 
+                        elif obj[2] != ' 'and float(obj[2]) > 1400000000.0:
+                            temp = obj[2]
+                        elif obj[3] != ' 'and float(obj[3]) > 1400000000.0:
+                            temp = obj[3]
+                        else:
+                            temp = 0
                         if index == 1:
-                            tFinAck = float(obj[1]) * 1000.0
+                            tFinAck = float(temp) * 1000.0
+                        elif index == 2:
+                            continue
                         elif index == 3:
-                            tAck = float(obj[1]) * 1000.0
+                            tAck = float(temp) * 1000.0
+                        else:
+                            tFinAck = 0
+                            tAck = 0
                     else:
                         main.log.error('Tshark output file for OFP' +
                             ' returned unexpected results')
@@ -571,7 +598,8 @@
                     main.log.info("ONOS " + str(nodeNum) + 
                             " end-to-end disconnection "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
+                            "threshold or premature iteration: ")
+                    main.log.info(str(endToEndDisc))
 
                 if finAckTransaction >= thresholdMin and\
                    finAckTransaction < thresholdMax and i >= iterIgnore:
@@ -583,8 +611,9 @@
                     main.log.info("ONOS "+str(nodeNum)+
                             " fin/ack transaction "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
-                
+                            "threshold or premature iteration: ")
+                    main.log.info(str(finAckTransaction))
+
                 if ackToDevice >= thresholdMin and\
                    ackToDevice < thresholdMax and i >= iterIgnore:
                     ackToDeviceLatNodeIter[node][i] = ackToDevice
@@ -595,8 +624,9 @@
                     main.log.info("ONOS "+str(nodeNum)+
                             " ack-to-device "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
-                
+                            "threshold or premature iteration: ")
+                    main.log.info(str(ackToDevice))
+
                 if deviceToGraph >= thresholdMin and\
                    deviceToGraph < thresholdMax and i >= iterIgnore:
                     deviceToGraphDiscLatNodeIter[node][i] = deviceToGraph
@@ -607,7 +637,8 @@
                     main.log.info("ONOS "+str(nodeNum)+
                             " device-to-graph disconnect "+
                             "measurement ignored due to excess in "+
-                            "threshold or premature iteration")
+                            "threshold or premature iteration: ")
+                    main.log.info(str(deviceToGraph))
 
         endToEndAvg = 0
         ofpToGraphAvg = 0
@@ -648,15 +679,15 @@
                     roleToOfpList.append(item)
 
             for item in roleReplyToDeviceLatNodeIter[node]:
-                if item > 0.0:
-                    tcpToFeatureList.append(item)
-
+                if item >= 0.0:
+                    ofpToDeviceList.append(item)
+            
             for item in featureToRoleRequestLatNodeIter[node]:
                 if item > 0.0:
                     featureToRoleList.append(item)
 
             for item in deviceToGraphLatNodeIter[node]:
-                if item > 0.0:
+                if item >= 0.0:
                     deviceToGraphList.append(item)
 
             # Switch disconnect measurements
@@ -673,9 +704,9 @@
                     ackToDeviceList.append(item)
 
             for item in deviceToGraphDiscLatNodeIter[node]:
-                if item > 0.0:
+                if item >= 0.0:
                     deviceToGraphDiscList.append(item)
-            
+
             endToEndAvg = round(numpy.mean(endToEndList), 2)
             endToEndStdDev = round(numpy.std(endToEndList), 2)
 
@@ -837,10 +868,10 @@
         downThresholdStr = main.params['TEST']['portDownThreshold']
         upThresholdObj = upThresholdStr.split(',')
         downThresholdObj = downThresholdStr.split(',')
-        upThresholdMin = float(upThresholdObj[0])
-        upThresholdMax = float(upThresholdObj[1])
-        downThresholdMin = float(downThresholdObj[0])
-        downThresholdMax = float(downThresholdObj[1])
+        upThresholdMin = int(upThresholdObj[0])
+        upThresholdMax = int(upThresholdObj[1])
+        downThresholdMin = int(downThresholdObj[0])
+        downThresholdMax = int(downThresholdObj[1])
         
         interfaceConfig = 's1-eth1'
         main.log.report('Port enable / disable latency')
@@ -876,6 +907,11 @@
                     interfaceConfig + ' down')
             main.Mininet1.handle.expect('mininet>')
             
+            jsonStrPtDown = []
+            for node in range (0, clusterCount): 
+                metricsPortDown = CLIs[node].topologyEventsMetrics()
+                jsonStrPtDown.append(metricsPortDown)
+            
             time.sleep(3)
             
             main.ONOS1.tsharkStop()
@@ -906,9 +942,9 @@
           
             for node in range(0, clusterCount):
                 nodeNum = node+1
-                metricsDown = CLIs[node].topologyEventsMetrics
-                jsonStrDown = metricsDown()
-                jsonObj = json.loads(jsonStrDown)
+                # metricsDown = CLIs[node].topologyEventsMetrics
+                #jsonStrPtDown[node] = metricsDown()
+                jsonObj = json.loads(jsonStrPtDown[node])
                 
                 if jsonObj:
                     graphTimestamp = jsonObj[graphTimestampKey]['value']
@@ -979,6 +1015,11 @@
             main.Mininet1.handle.sendline('sh ifconfig ' + interfaceConfig + ' up')
             main.Mininet1.handle.expect('mininet>')
             
+            jsonStrPtUp = []
+            for node in range (0, clusterCount): 
+                metricsPortUp = CLIs[node].topologyEventsMetrics()
+                jsonStrPtUp.append(metricsPortUp)
+            
             time.sleep(5)
             main.ONOS1.tsharkStop()
             
@@ -1001,9 +1042,9 @@
             
             for node in range(0, clusterCount):
                 nodeNum = node+1
-                metricsUp = CLIs[node].topologyEventsMetrics
-                jsonStrUp = metricsUp()
-                jsonObj = json.loads(jsonStrUp)
+                #metricsUp = CLIs[node].topologyEventsMetrics
+                #jsonStrUp = metricsUp()
+                jsonObj = json.loads(jsonStrPtUp[node])
                
                 if jsonObj:
                     graphTimestamp = jsonObj[graphTimestampKey]['value']
@@ -1102,10 +1143,9 @@
                     portUpDevToLinkList.append(item)
 
             for item in portUpLinkToGraphNodeIter[node]:
-                if item > 0.0:
+                if item >= 0.0:
                     portUpLinkToGraphList.append(item)
 
-
             for item in portDownEndToEndNodeIter[node]:
                 if item > 0.0:
                     portDownEndToEndList.append(item)
@@ -1115,11 +1155,11 @@
                     portDownOfpToDevList.append(item)
 
             for item in portDownDevToLinkNodeIter[node]:
-                if item > 0.0:
+                if item >= 0.0:
                     portDownDevToLinkList.append(item)
 
             for item in portDownLinkToGraphNodeIter[node]:
-                if item > 0.0:
+                if item >= 0.0:
                     portDownLinkToGraphList.append(item)
 
             portUpEndToEndAvg = round(numpy.mean(portUpEndToEndList), 2)
@@ -1149,7 +1189,7 @@
                     str(round(portDownEndToEndAvg, 2)) + ' ms')
             main.log.report(' Port down Ofp-to-device ' +
                     str(portDownOfpToDevAvg) + ' ms')
-            main.log.report(' Port down Device-to-link' +
+            main.log.report(' Port down Device-to-link ' +
                     str(portDownDevToLinkAvg) + ' ms')
             main.log.report(' Port down Link-to-graph' +
                     str(portDownLinkToGraphAvg) + ' ms')
diff --git a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.topo b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.topo
index 6a70f60..570dece 100644
--- a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.topo
+++ b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.topo
@@ -2,7 +2,7 @@
     <COMPONENT>
         
         <ONOSbench>
-            <host>10.128.174.10</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -11,7 +11,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.174.10</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -20,7 +20,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.174.10</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -29,7 +29,7 @@
         </ONOS2cli>
         
         <ONOS3cli>
-            <host>10.128.174.10</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -38,7 +38,7 @@
         </ONOS3cli>
         
         <ONOS4cli>
-            <host>10.128.174.10</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -47,7 +47,7 @@
         </ONOS4cli>
         
         <ONOS5cli>
-            <host>10.128.174.10</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -56,7 +56,7 @@
         </ONOS5cli>
         
         <ONOS6cli>
-            <host>10.128.174.10</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -65,7 +65,7 @@
         </ONOS6cli>
         
         <ONOS7cli>
-            <host>10.128.174.10</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -74,9 +74,9 @@
         </ONOS7cli>
 
         <ONOS1>
-            <host>10.128.174.1</host>
-            <user>sdn</user>
-            <password>rocks</password>
+            <host>10.254.1.201</host>
+            <user>admin</user>
+            <password>onos_test</password>
             <type>OnosDriver</type>
             <connect_order>9</connect_order>
             <COMPONENTS> </COMPONENTS>
@@ -84,8 +84,8 @@
 
         <ONOS2>
             <host>10.254.1.202</host>
-            <user>sdn</user>
-            <password>rocks</password>
+            <user>admin</user>
+            <password>onos_test</password>
             <type>OnosDriver</type>
             <connect_order>10</connect_order>
             <COMPONENTS> </COMPONENTS>
@@ -93,8 +93,8 @@
 
         <ONOS3>
             <host>10.254.1.203</host>
-            <user>sdn</user>
-            <password>rocks</password>
+            <user>admin</user>
+            <password>onos_test</password>
             <type>OnosDriver</type>
             <connect_order>11</connect_order>
             <COMPONENTS> </COMPONENTS>
@@ -102,8 +102,8 @@
         
         <ONOS4>
             <host>10.254.1.204</host>
-            <user>sdn</user>
-            <password>rocks</password>
+            <user>admin</user>
+            <password>onos_test</password>
             <type>OnosDriver</type>
             <connect_order>12</connect_order>
             <COMPONENTS> </COMPONENTS>
@@ -111,8 +111,8 @@
         
         <ONOS5>
             <host>10.254.1.205</host>
-            <user>sdn</user>
-            <password>rocks</password>
+            <user>admin</user>
+            <password>onos_test</password>
             <type>OnosDriver</type>
             <connect_order>13</connect_order>
             <COMPONENTS> </COMPONENTS>
@@ -120,8 +120,8 @@
         
         <ONOS6>
             <host>10.254.1.206</host>
-            <user>sdn</user>
-            <password>rocks</password>
+            <user>admin</user>
+            <password>onos_test</password>
             <type>OnosDriver</type>
             <connect_order>14</connect_order>
             <COMPONENTS> </COMPONENTS>
@@ -129,8 +129,8 @@
         
         <ONOS7>
             <host>10.254.1.207</host>
-            <user>sdn</user>
-            <password>rocks</password>
+            <user>admin</user>
+            <password>onos_test</password>
             <type>OnosDriver</type>
             <connect_order>15</connect_order>
             <COMPONENTS> </COMPONENTS>
@@ -144,7 +144,7 @@
             <connect_order>16</connect_order>
             <COMPONENTS>
                 <arg1> --custom topo-perf-2sw.py </arg1>
-                <arg2> --arp --mac --topo mytopo</arg2>
+                <arg2> --topo mytopo</arg2>
                 <arg3> </arg3>
                 <controller> remote </controller>
             </COMPONENTS>
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/TopoPerfNextBM/__init__.py
similarity index 100%
rename from TestON/tests/TopoPerfNext/__init__.py
rename to TestON/tests/TopoPerfNextBM/__init__.py
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.params b/TestON/tests/TopoPerfNextBM/backup/TopoPerfNextBM.params
similarity index 77%
rename from TestON/tests/TopoPerfNext/TopoPerfNext.params
rename to TestON/tests/TopoPerfNextBM/backup/TopoPerfNextBM.params
index dc2d519..d681433 100644
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.params
+++ b/TestON/tests/TopoPerfNextBM/backup/TopoPerfNextBM.params
@@ -1,8 +1,9 @@
 <PARAMS>
-    <testcases>1,2,3,6,2,3,6,2,3,6,2,3</testcases>
+    <testcases>1,2,3,4,2,3,4,2,3,4,2,3</testcases>
 
     <ENV>
         <cellName>topo_perf_test</cellName>
+        <cellFeatures>"org.onosproject.metrics,org.onosproject.openflow"</cellFeatures>
     </ENV>
 
     <GIT>
@@ -12,7 +13,7 @@
     </GIT>
 
     <CTRL>
-        <user>admin</user>
+        <user>sdn</user>
         <ip1>10.128.174.1</ip1>
         <port1>6633</port1>
         <ip2>10.128.174.2</ip2>
@@ -35,6 +36,15 @@
         <ip>10.128.174.10</ip>
     </BENCH>
 
+    <TSHARK>
+        <ofpPortStatus>OF 1.3 146</ofpPortStatus>
+        <ofpRoleReply>OF 1.3 90 of_role_reply</ofpRoleReply>
+        <featureReply>OF 1.3 98 of_features_reply</featureReply>
+        <roleRequest>OF 1.3 90 of_role_request</roleRequest>
+        <tcpSynAck>TCP 74 6633</tcpSynAck>
+        <finAckSequence>FIN</finAckSequence>
+    </TSHARK>
+
     <TEST>
         #'on' or 'off' debug mode.
         #If on, logging will be more verbose and
@@ -70,10 +80,10 @@
     <DB>
         <postToDB>on</postToDB>
         <portEventResultPath>
-        /home/admin/ONLabTest/TestON/tests/TopoPerfNext/portEventResultDb.log
+        /home/admin/ONLabTest/TestON/tests/TopoPerfNextBM/portEventResultDb.log
         </portEventResultPath>
         <switchEventResultPath>
-        /home/admin/ONLabTest/TestON/tests/TopoPerfNext/switchEventResultDb.log
+        /home/admin/ONLabTest/TestON/tests/TopoPerfNextBM/switchEventResultDb.log
         </switchEventResultPath>
     </DB>
 
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/TopoPerfNextBM/portEventResultDb.log
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/TopoPerfNextBM/portEventResultDb.log
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/TopoPerfNextBM/switchEventResultDb.log
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/TopoPerfNextBM/switchEventResultDb.log
diff --git a/TestON/tests/flowTP1g/flowTP1g.params b/TestON/tests/flowTP1g/flowTP1g.params
index 0f08c9a..54320a7 100644
--- a/TestON/tests/flowTP1g/flowTP1g.params
+++ b/TestON/tests/flowTP1g/flowTP1g.params
@@ -1,26 +1,27 @@
 <PARAMS>
 
-    <testcases>1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2</testcases>
     
     <isOnBaremetal>False</isOnBaremetal>
-    <SCALE>3</SCALE>
+    <SCALE>1,3,3,5,5,7,7</SCALE>
     <availableNodes>7</availableNodes>
     
 
 
     <ENV>
         <cellName>flowTP</cellName>
-        <cellApps>null,demo</cellApps>
+        <cellApps>drivers,null,demo</cellApps>
     </ENV>
 
     <TEST>
+        <enableFlowRuleStoreBackup>true</enableFlowRuleStoreBackup>
         <skipCleanInstall>yes</skipCleanInstall> 
-        <warmUp>4</warmUp>                              #number of runs to warm up the system
-        <sampleSize>3</sampleSize>                     #number of runs to take data from
+        <warmUp>5</warmUp>                              #number of runs to warm up the system
+        <sampleSize>20</sampleSize>                     #number of runs to take data from
         <neighbors>0,a</neighbors>                      #list of number of neighbors, a = all 
-        <flows>3500</flows>
+        <flows>122500</flows>
         <switches>35</switches>
-
+        <cooldown>10</cooldown>
         <testCMD0>flow-tester.py -f</testCMD0>          #base command 
         <testCMD1> -n </testCMD1>                       #neighbors 
         
@@ -46,24 +47,24 @@
         <ip4>10.128.5.54</ip4>
         <port4>6633</port4>
         
-        <ip5>10.128.5.65</ip5>
+        <ip5>10.128.5.55</ip5>
         <port5>6633</port5>
         
-        <ip6>10.128.5.66</ip6>
+        <ip6>10.128.5.56</ip6>
         <port6>6633</port6> 
        
-        <ip7>10.128.5.67</ip7>
+        <ip7>10.128.5.57</ip7>
         <port7>6633</port7>
 
     </CTRL>
 
     <MN>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.128.5.59</ip1>
     </MN>
 
     <BENCH>
         <user>admin</user>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.128.5.50</ip1>
     </BENCH>
 
     <JSON>
diff --git a/TestON/tests/flowTP1g/flowTP1g.py b/TestON/tests/flowTP1g/flowTP1g.py
index 352341e..cbea039 100644
--- a/TestON/tests/flowTP1g/flowTP1g.py
+++ b/TestON/tests/flowTP1g/flowTP1g.py
@@ -34,9 +34,12 @@
         maxNodes = int(main.params[ 'availableNodes' ])
         skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
         cellName = main.params[ 'ENV' ][ 'cellName' ]       
- 
-        #----do i need this line?
-        #main.ONOSbench.handle.sendline("export TERM=vt100")
+    
+        main.log.info("==========DEBUG VERSION 3===========")
+
+        main.exceptions = [0]*11
+        main.warnings = [0]*11
+        main.errors = [0]*11
 
         # -- INIT SECTION, ONLY RUNS ONCE -- #
         if init == False:
@@ -132,7 +135,7 @@
             a(ONOSIp[node])
          
         main.log.info("Startup sequence complete")
-
+        main.ONOSbench.onosErrorLog(ONOSIp[1])
         
     def CASE2( self, main ):
         #
@@ -145,6 +148,17 @@
         import datetime
         import traceback
 
+        global currentNeighbors
+        try:
+            currentNeighbors
+        except:
+            currentNeighbors = "0"
+        else:
+            if currentNeighbors == "r":      #reset
+                currentNeighbors = "0"
+            else:
+                currentNeighbors = "a"
+
         testCMD = [ 0,0,0,0 ]
         warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
         sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
@@ -154,30 +168,33 @@
         testCMD[1] = main.params[ 'TEST' ][ 'testCMD1' ]
         maxNodes = main.params[ 'availableNodes' ]
         onBaremetal = main.params['isOnBaremetal']
-
+        cooldown = main.params[ 'TEST' ][ 'cooldown' ]
         cellName = main.params[ 'ENV' ][ 'cellName' ]
         BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
         BENCHUser = main.params[ 'BENCH' ][ 'user' ]
         MN1Ip = main.params[ 'MN' ][ 'ip1' ]
         maxNodes = int(main.params[ 'availableNodes' ])
         homeDir = os.path.expanduser('~')
-  
+        flowRuleBackup = str(main.params[ 'TEST' ][ 'enableFlowRuleStoreBackup' ])
+        main.log.info("Flow Rule Backup is set to:" + flowRuleBackup)
+
         servers = str(clusterCount) 
-        for i in range(0, len(neighborList)):
-            if neighborList[i] == 'a':
-                neighborList[i] = str(clusterCount - 1)  
         
         if clusterCount == 1: 
             neighborList = ['0']
+            currentNeighbors = "r"
+        else:
+            if currentNeighbors == "a":
+                neighborList = [str(clusterCount-1)]
+                currentNeighbors = "r"
+            else:
+                neighborList = ['0'] 
+        
         main.log.info("neightborlist: " + str(neighborList))
 
         ts = time.time()
         st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
 
-        #initialize log file, remove any previous data
-        resultsLog = open("flowTP1gDB","w+")
-        resultsLog.close()
-
         #write file to change mem limit to 32 gigs (BAREMETAL ONLY!)
         if onBaremetal == "true":
             filename = "/onos/tools/package/bin/onos-service"
@@ -208,25 +225,33 @@
                 ipCSV += main.params[ 'CTRL' ][ tempstr ] 
                 if i < int(maxNodes):
                     ipCSV +=","
-            
-            filename = "/onos/tools/package/etc/org.onosproject.provider.nil.link.impl.NullLinkProvider.cfg"
-            linkConfig = open(homeDir + filename,'w+')
-            linkConfig.write("# eventRate = 2000\n")
-            linkConfig.write("neighbors = "  + ipCSV)
-            main.log.info(" NullLinkProvider.cfg: " + ipCSV)
-            linkConfig.close()
-            
-            main.ONOSbench.handle.sendline("""onos $OC1 "cfg setorg.onosproject.provider.nil.NullProviders enabled true" """)
-            main.ONOSbench.handle.expect(":~")
+           
+
             main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount 35" """)
             main.ONOSbench.handle.expect(":~")
+            time.sleep(3)
             main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
             main.ONOSbench.handle.expect(":~")
+            time.sleep(3)
             main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
             main.ONOSbench.handle.expect(":~")
+            time.sleep(3)
             main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
             main.ONOSbench.handle.expect(":~")
-        
+            time.sleep(3)
+            main.log.info("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBackup + """" """)
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBackup + """" """)
+            main.ONOSbench.handle.expect(":~")
+       
+            main.ONOSbench.handle.sendline("onos $OC1 summary")
+            main.ONOSbench.handle.expect(":~")
+            check = main.ONOSbench.handle.before
+
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg get" """)
+            main.ONOSbench.handle.expect(":~")
+            check = main.ONOSbench.handle.before
+            main.log.info("\nStart up check: \n" + check + "\n") 
+
             #devide flows
             flows = int(main.params[ 'TEST' ][ 'flows' ])
             main.log.info("Flow Target  = " + str(flows))
@@ -243,35 +268,51 @@
             data = [[""]*int(servers)]*int(sampleSize)
             maxes = [""]*int(sampleSize)
 
-            for test in range(0, (warmUp + sampleSize)): 
-                flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
-                flowCMD += testCMD[0] + " " + str(flows) + " " + testCMD[1] 
-                flowCMD += " " + str(n) + " " + str(serverEnvVars)
-                print("\n")                    
-                main.log.info("COMMAND: " + flowCMD)
-                main.log.info("Executing command") 
-                
-                for i in range(0,15):
-                    main.ONOSbench.handle.sendline(flowCMD)
-                    time.sleep(1)
-                    main.ONOSbench.handle.expect(":~")
-                    rawResult = main.ONOSbench.handle.before
-                    if " -> " in rawResult:
-                        break
+            flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
+            flowCMD += testCMD[0] + " " + str(flows) + " " + testCMD[1]
+            flowCMD += " " + str(n) + " " + str(serverEnvVars) + "-j" 
 
+            main.log.info(flowCMD)
+            #time.sleep(60)
+            
+            for test in range(0, warmUp + sampleSize): 
+                if test < warmUp: 
+                    main.log.info("Warm up " + str(test + 1) + " of " + str(warmUp))
+                else: 
+                     main.log.info("====== Test run: " + str(test-warmUp+1) + " ======") 
+
+                main.ONOSbench.handle.sendline(flowCMD)
+                main.ONOSbench.handle.expect(":~")
+                rawResult = main.ONOSbench.handle.before
+                main.log.info("Raw results: \n" + rawResult + "\n")
+
+                if "failed" in rawResult: 
+                    main.log.report("FLOW_TESTER.PY FAILURE")
+                    main.log.report( " \n" + rawResult + " \n") 
+                    break
+            
+            ########################################################################################
                 result = [""]*(clusterCount)
                 rawResult = rawResult.splitlines()
-                print(rawResult)
+
                 for node in range(1, clusterCount + 1):        
                     for line in rawResult:
-                        if ONOSIp[node] in line and " -> " in line:
-                            myLine = line.split(" ")
-                            for word in myLine:
-                                if "ms" in word:
-                                    result[node-1] = int(word.replace("ms",""))
-                                    main.log.info("Parsed result: " + str(result[node-1]))
-                                    break
-                            break
+                        #print("line: " + line) 
+                        if ONOSIp[node] in line and "server" in line:
+                            temp = line.split(" ") 
+                            for word in temp:
+                                #print ("word: " + word) 
+                                if "elapsed" in repr(word): 
+                                    #print("in elapsed ==========")
+                                    index = temp.index(word) + 1
+                                    #print ("index: " + str(index)) 
+                                    #print ("temp[index]: " + temp[index])
+                                    myParsed = (temp[index]).replace(",","")
+                                    myParsed = myParsed.replace("}","")
+                                    myParsed = int(myParsed)
+                                    result[node-1] = myParsed
+                                    main.log.info( ONOSIp[node] + " : " + str(myParsed))
+                                    break 
 
                 if test >= warmUp:
                     for i in result: 
@@ -283,40 +324,27 @@
                     main.log.info("Data collection iteration: " + str(test-warmUp) + " of " + str(sampleSize))
                     main.log.info("Throughput time: " + str(maxes[test-warmUp]) + "(ms)")                
 
-                if test >= warmUp:
                     data[test-warmUp] = result
 
                 # wait for flows = 0 
-                removedFlows = False
-                repeat = 0
-                time.sleep(3)
-                while ( removedFlows == False and repeat <= 10 ):
-                    main.ONOSbench.handle.sendline("onos $OC1 summary| cut -d ' ' -f6")
-                    main.ONOSbench.handle.expect("~")
-                    before = main.ONOSbench.handle.before
-                    parseTest = before.splitlines()
-                    flowsummary = ""
-                    for line in parseTest:
-                        if "flow" in str(line):
-                            flowsummary = line 
-                            break
-                    currentflow = ""
-                    for word in flowsummary.split(" "): 
-                        if "flow" in str(word):
-                            currentflow = str(word)
-                    currentflow = currentflow.replace(",","")
-                    currentflow = currentflow.replace("\n","")
-                    #main.log.info(currentflow)
+                for checkCount in range(0,5): 
+                    time.sleep(10)
+                    main.ONOSbench.handle.sendline("onos $OC1 summary")
+                    main.ONOSbench.handle.expect(":~")
+                    flowCheck = main.ONOSbench.handle.before
+                    if "flows=0," in flowCheck: 
+                        main.log.info("Flows removed")
+                        break
+                    else: 
+                        for line in flowCheck.splitlines(): 
+                            if "flows=" in line: 
+                                main.log.info("Current Summary: " + line) 
+                    if checkCount == 2: 
+                        main.log.info("Flows are stuck, moving on ")
 
-                    zeroFlow = "flows=0"                
-                    if zeroFlow in before:
-                        removedFlows = True 
-                        main.log.info("\t Wait 5 sec of cool down...")
-                        time.sleep(5)
 
-                    time.sleep(5)
-                    repeat +=1
-     
+                time.sleep(5)
+                
             main.log.info("raw data: " + str(data))
             main.log.info("maxes:" + str(maxes))
 
@@ -361,12 +389,15 @@
             main.log.info("Standard deviation of throughput: " + str(stdTP) + " Kflows/second") 
 
             resultsLog = open("flowTP1gDB","a")
-            resultsLog.write("'" + commit + "',")
-            resultsLog.write("'1gig',")
-            resultsLog.write((main.params[ 'TEST' ][ 'flows' ]) + ",")
-            resultsLog.write(str(clusterCount)+ ",")
-            resultsLog.write(str(n) + ",")
-            resultsLog.write(str(avgTP) + "," + str(stdTP) + "\n")
+            resultString = ("'" + commit + "',")
+            resultString += ("'1gig',")
+            resultString += ((main.params[ 'TEST' ][ 'flows' ]) + ",")
+            resultString += (str(clusterCount) + ",")
+            resultString += (str(n) + ",")
+            resultString += (str(avgTP) + "," + str(stdTP) + "\n")
+            resultsLog.write(resultString) 
             resultsLog.close()
-
-             
+            
+            main.log.report("Result line to file: " + resultString)
+           
+        main.ONOSbench.onosErrorLog(ONOSIp[1]) 
diff --git a/TestON/tests/flowTP1g/flowTP1g.topo b/TestON/tests/flowTP1g/flowTP1g.topo
index 30a1467..763a4d6 100644
--- a/TestON/tests/flowTP1g/flowTP1g.topo
+++ b/TestON/tests/flowTP1g/flowTP1g.topo
@@ -3,7 +3,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -12,7 +12,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
         </ONOS2cli>
 
         <ONOS3cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
         </ONOS3cli>
 
         <ONOS4cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
         </ONOS4cli>
 
         <ONOS5cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
         </ONOS5cli>
 
         <ONOS6cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
         </ONOS6cli>
 
         <ONOS7cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -112,7 +112,7 @@
 
     
         <ONOS5>
-            <host>10.128.5.65</host>
+            <host>10.128.5.55</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -121,7 +121,7 @@
         </ONOS5>
 
         <ONOS6>
-            <host>10.128.5.66</host>
+            <host>10.128.5.56</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -130,7 +130,7 @@
         </ONOS6>
 
         <ONOS7>
-            <host>10.128.5.67</host>
+            <host>10.128.5.57</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -141,6 +141,4 @@
     </COMPONENT>
 
 </TOPOLOGY>
-
-
-    
+