Merge pull request #116 from opennetworkinglab/devl/HA_changes

Add tests for distributed sets and counters
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 1f153ee..a78f4f2 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -1901,3 +1901,147 @@
             main.cleanup()
             main.exit()
 
+
+    def getOnosIpFromEnv(self):
+
+        import string  
+
+        # returns a list of ip addresses for the onos nodes, will work with up to 7 nodes + OCN and OCI
+        # returns in format [ 'x', OC1 ip, OC2 i... ect. ... , ONN ip ]
+
+        self.handle.sendline("env| grep OC") 
+        self.handle.expect(":~")
+        rawOutput = self.handle.before
+        print rawOutput
+        print "-----------------------------"
+        print repr(rawOutput)
+        mpa = dict.fromkeys(range(32))
+        translated = rawOutput.translate(mpa)
+        print translated
+
+
+        # create list with only the lines that have the needed IPs 
+        unparsedIps = []
+
+        # remove preceeding or trailing lines
+        for line in rawOutput: 
+            if "OC" in line and "=" in line: 
+                unparsedIps.append(str(line)) 
+
+        # determine cluster size
+        clusterCount = 0
+        for line in unparsedIps:
+            line = str(line)
+            print line
+            temp = line.replace("OC","")
+            print("line index " + str(line.index("="))) 
+            OCindex = temp[0]
+            for i in range(0, 7):
+                if OCindex == str(i) and i > clusterCount:
+                    clusterCount == i
+                    print(clusterCount)
+        # create list to hold ips such that OC1 is at list[1] and OCN and OCI are at the end (in that order)
+        ONOSIps = ["x"] * (clusterCount + 3) 
+
+        # populate list 
+        for line in unparsedIps:
+            main.log.info(line)########## 
+            temp = str(line.replace("OC",""))
+            main.log.info(str(list(temp)))
+            OCindex = temp[0]
+            main.log.info(OCindex)############
+            if OCindex == "N":
+                ONOSIps[ clusterCount + 1 ] = temp.replace("N=","")
+    
+            if OCindex == "I":
+                ONOSIps[ clusterCount + 2 ] = temp.replace("I=","")
+
+            else:
+                ONOSIps[ int(OCindex) ] = temp.replace((OCindex + "=") ,"")
+
+        # validate 
+        for x in ONOSIps: 
+            if ONOSIps.index(x) != 0 and x == "x": 
+                main.log.error("ENV READ FAILURE, MISSING DATA: \n\n" + str(ONOSIps) + "\n\n") 
+
+        return ONOSIps
+
+        
+    def onosErrorLog(self, nodeIp): 
+
+        cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep WARN"
+        self.handle.sendline(cmd) 
+        self.handle.expect(":~")
+        before = (self.handle.before).splitlines() 
+    
+        warnings = []
+
+        for line in before: 
+            if "WARN" in line and "grep" not in line: 
+                warnings.append(line) 
+                main.warnings[main.warnings[0]+1] = line
+                main.warnings[0] += 1
+                if main.warnings[0] >= 10: 
+                    main.warnings[0] = 0 
+
+        cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep ERROR"
+        self.handle.sendline(cmd)
+        self.handle.expect(":~")
+        before = (self.handle.before).splitlines()
+
+        errors = []
+
+        for line in before:
+            if "ERROR" in line and "grep" not in line:
+                errors.append(line)
+                main.errors[main.errors[0]+1] = line
+                main.errors[0] += 1
+                if main.errors[0] >= 10:
+                    main.errors[0] = 0
+
+        cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep Exept"
+        self.handle.sendline(cmd)
+        self.handle.expect(":~")
+        before = (self.handle.before).splitlines()
+
+        exceptions = []
+
+        for line in before:
+            if "Except" in line and "grep" not in line:
+                exceptions.append(line)
+                main.exceptions[main.errors[0]+1] = line
+                main.exceptions[0] += 1
+                if main.exceptions[0] >= 10:
+                    main.exceptions[0] = 0
+
+
+
+        ################################################################
+
+        msg1 = "WARNINGS: \n"
+        for i in main.warnings: 
+            if type(i) is not int: 
+                msg1 += ( i + "\n")
+
+        msg2 = "ERRORS: \n"
+        for i in main.errors:
+            if type(i) is not int:
+                msg2 += ( i + "\n")
+
+        msg3 = "EXCEPTIONS: \n"
+        for i in main.exceptions: 
+            if type(i) is not int:
+                msg3 += ( i + "\n")
+
+        main.log.info("===============================================================\n") 
+        main.log.info( "Warnings: " + str(len(warnings))) 
+        main.log.info( "Errors: " + str(len(errors))) 
+        main.log.info( "Exceptions: " + str(len(exceptions)))
+        if len(warnings) > 0:
+            main.log.info(msg1)
+        if len(errors) > 0: 
+            main.log.info(msg2)
+        if len(exceptions) > 0: 
+            main.log.info(msg3)
+        main.log.info("===============================================================\n")
+        
diff --git a/TestON/tests/FuncIntent/FuncIntent.params b/TestON/tests/FuncIntent/FuncIntent.params
new file mode 100755
index 0000000..3d2ef8a
--- /dev/null
+++ b/TestON/tests/FuncIntent/FuncIntent.params
@@ -0,0 +1,34 @@
+<PARAMS>
+
+    <testcases>10,11,12,1001,10,12,1001</testcases>
+    
+    <SCALE>1,3</SCALE>
+    <availableNodes>3</availableNodes>
+    <ENV>
+        <cellName>single_func</cellName>
+        <cellApps>drivers,openflow,proxyarp,mobility</cellApps>
+    </ENV>
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+    <CTRL>
+        <num>3</num>
+        <ip1>OC1</ip1>
+        <port1>6633</port1>
+        <ip2>OC2</ip2>
+        <port2>6633</port2>
+        <ip3>OC3</ip3>
+        <port3>6633</port3>
+    </CTRL>
+    <BENCH>
+        <user>admin</user>
+        <ip1>10.128.10.20</ip1>
+    </BENCH>
+    <MININET>
+        <switch>7</switch>
+        <links>20</links>
+        <topo>~/mininet/custom/newFuncTopo.py</topo>
+    </MININET>
+
+</PARAMS>
diff --git a/TestON/tests/FuncIntent/FuncIntent.py b/TestON/tests/FuncIntent/FuncIntent.py
new file mode 100644
index 0000000..1db5752
--- /dev/null
+++ b/TestON/tests/FuncIntent/FuncIntent.py
@@ -0,0 +1,501 @@
+
+# Testing the basic functionality of ONOS Next
+# For sanity and driver functionality excercises only.
+
+import time
+import json
+
+time.sleep( 1 )
+
+class FuncIntent:
+
+    def __init__( self ):
+        self.default = ''
+
+    def CASE10( self, main ):
+        import time
+        import os
+        """
+        Startup sequence:
+        cell <name>
+        onos-verify-cell
+        onos-remove-raft-log
+        git pull
+        mvn clean install
+        onos-package
+        onos-install -f
+        onos-wait-for-start
+        """
+        global init
+        try:
+            if type(init) is not bool:
+                init = False
+        except NameError:
+            init = False
+        #Local variables
+        cellName = main.params[ 'ENV' ][ 'cellName' ]
+        apps = main.params[ 'ENV' ][ 'cellApps' ]
+        gitBranch = main.params[ 'GIT' ][ 'branch' ]
+        #benchIp = main.params[ 'BENCH' ][ 'ip1' ]
+        benchUser = main.params[ 'BENCH' ][ 'user' ]
+        topology = main.params[ 'MININET' ][ 'topo' ]
+        #maxNodes = int( main.params[ 'availableNodes' ] )
+        main.numSwitch = int( main.params[ 'MININET' ][ 'switch' ] )
+        main.numLinks = int( main.params[ 'MININET' ][ 'links' ] )
+        main.numCtrls = main.params[ 'CTRL' ][ 'num' ]
+        PULLCODE = False
+        if main.params[ 'GIT' ][ 'pull' ] == 'True':
+            PULLCODE = True
+        main.case( "Setting up test environment" )
+        main.CLIs = []
+        for i in range( 1, int( main.numCtrls ) + 1 ):
+            main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+
+        # -- INIT SECTION, ONLY RUNS ONCE -- #
+        if init == False:
+            init = True
+
+            #main.ONOSip = []
+            main.ONOSport = []
+            main.scale = ( main.params[ 'SCALE' ] ).split( "," )
+            main.numCtrls = int( main.scale[ 0 ] )
+
+            if PULLCODE:
+                main.step( "Git checkout and pull " + gitBranch )
+                main.ONOSbench.gitCheckout( gitBranch )
+                gitPullResult = main.ONOSbench.gitPull()
+                if gitPullResult == main.ERROR:
+                    main.log.error( "Error pulling git branch" )
+                main.step( "Using mvn clean & install" )
+                cleanInstallResult = main.ONOSbench.cleanInstall()
+                stepResult = cleanInstallResult
+                utilities.assert_equals( expect=main.TRUE,
+                                         actual=stepResult,
+                                         onpass="Successfully compiled " +
+                                                "latest ONOS",
+                                         onfail="Failed to compile " +
+                                                "latest ONOS" )
+            else:
+                main.log.warn( "Did not pull new code so skipping mvn " +
+                               "clean install" )
+            # Populate main.ONOSip with ips from params
+            #for i in range( 1, maxNodes + 1):
+            #    main.ONOSip.append( main.params[ 'CTRL' ][ 'ip' + str( i ) ] )
+            #    main.ONOSport.append( main.params[ 'CTRL' ][ 'port' +
+            #                          str( i ) ])
+        
+            ONOSip = main.ONOSbench.getOnosIpFromEnv()
+            main.log.info("\t\t CLUSTER IPs: \n\n" + str(ONOSip) + "\n\n") 
+
+        main.numCtrls = int( main.scale[ 0 ] )
+        main.scale.remove( main.scale[ 0 ] )
+        
+        #kill off all onos processes
+        main.log.info( "Safety check, killing all ONOS processes" +
+                       " before initiating enviornment setup" )
+        for i in range( maxNodes ):
+            main.ONOSbench.onosDie( main.ONOSip[ i ] )
+        """
+        main.step( "Apply cell to environment" )
+        cellResult = main.ONOSbench.setCell( cellName )
+        verifyResult = main.ONOSbench.verifyCell()
+        stepResult = cellResult and verifyResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully applied cell to " + \
+                                        "environment",
+                                 onfail="Failed to apply cell to environment " )
+        """
+        """main.step( "Removing raft logs" )
+        removeRaftResult = main.ONOSbench.onosRemoveRaftLogs()
+        stepResult = removeRaftResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully removed raft logs",
+                                 onfail="Failed to remove raft logs" )
+        """
+        print "NODE COUNT = ", main.numCtrls
+        main.log.info( "Creating cell file" )
+        cellIp = []
+        for i in range( main.numCtrls ):
+            cellIp.append( str( main.ONOSip[ i ] ) )
+        print cellIp
+        main.ONOSbench.createCellFile( benchIp, cellName, "",
+                                       str( apps ), *cellIp )
+
+        main.step( "Apply cell to environment" )
+        cellResult = main.ONOSbench.setCell( cellName )
+        verifyResult = main.ONOSbench.verifyCell()
+        stepResult = cellResult and verifyResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully applied cell to " + \
+                                        "environment",
+                                 onfail="Failed to apply cell to environment " )
+
+        main.step( "Creating ONOS package" )
+        packageResult = main.ONOSbench.onosPackage()
+        stepResult = packageResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully created ONOS package",
+                                 onfail="Failed to create ONOS package" )
+
+        main.step( "Uninstalling ONOS package" )
+        onosUninstallResult = main.TRUE
+        for i in range( main.numCtrls):
+            onosUninstallResult = onosUninstallResult and \
+                    main.ONOSbench.onosUninstall( nodeIp=main.ONOSip[ i ] )
+        stepResult = onosUninstallResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully uninstalled ONOS package",
+                                 onfail="Failed to uninstall ONOS package" )
+        time.sleep( 5 )
+        main.step( "Installing ONOS package" )
+        onosInstallResult = main.TRUE
+        for i in range( main.numCtrls):
+            onosInstallResult = onosInstallResult and \
+                    main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+        stepResult = onosInstallResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully installed ONOS package",
+                                 onfail="Failed to install ONOS package" )
+
+        time.sleep( 5 )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+        
+        main.step( "Start ONOS cli" )
+        cliResult = main.TRUE
+        for i in range( main.numCtrls ):
+            cliResult = cliResult and \
+                        main.CLIs[i].startOnosCli( main.ONOSip[ i ] )
+        stepResult = cliResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully start ONOS cli",
+                                 onfail="Failed to start ONOS cli" )
+
+    def CASE11( self, main ):
+        """
+            Start mininet
+        """
+        main.log.report( "Start Mininet topology" )
+        main.log.case( "Start Mininet topology" )
+
+        main.step( "Starting Mininet Topology" )
+        topoResult = main.Mininet1.startNet( topoFile=topology )
+        stepResult = topoResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully loaded topology",
+                                 onfail="Failed to load topology" )
+        # Exit if topology did not load properly
+        if not topoResult:
+            main.cleanup()
+            main.exit()
+
+    def CASE12( self, main ):
+        """
+            Assign mastership to controllers
+        """
+        import re
+
+        main.step( "Assigning switches to controllers" )
+        assignResult = main.TRUE
+        for i in range( 1, ( main.numSwitch + 1 ) ):
+            main.Mininet1.assignSwController( sw=str( i ),
+                                              count=1,
+                                              ip1=main.ONOSip[ 0 ],
+                                              port1=main.ONOSport[ 0 ] )
+        for i in range( 1, ( main.numSwitch + 1 ) ):
+            response = main.Mininet1.getSwController( "s" + str( i ) )
+            print( "Response is " + str( response ) )
+            if re.search( "tcp:" + main.ONOSip[ 0 ], response ):
+                assignResult = assignResult and main.TRUE
+            else:
+                assignResult = main.FALSE
+        stepResult = assignResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully assigned switches" +
+                                        "to controller",
+                                 onfail="Failed to assign switches to " +
+                                        "controller" )
+
+    def CASE1001( self, main ):
+        """
+            Add host intents between 2 host:
+                - Discover hosts
+                - Add host intents
+                - Check intents
+                - Check flows
+                - Ping hosts
+                - Reroute
+                    - Link down
+                    - Ping hosts
+                    - Link up
+                    - Ping hosts
+                - Remove intents
+        """
+        import time
+        import json
+        import re
+        """
+            Create your item(s) here
+            item = { 'name': '', 'host1':
+                     { 'name': '', 'MAC': '00:00:00:00:00:0X',
+                       'id':'00:00:00:00:00:0X/-X' } , 'host2':
+                     { 'name': '', 'MAC': '00:00:00:00:00:0X',
+                       'id':'00:00:00:00:00:0X/-X'}, 'link': { 'switch1': '',
+                       'switch2': '', 'num':'' } }
+        """
+        # Local variables
+        items = []
+        ipv4 = { 'name': 'IPV4', 'host1':
+                 { 'name': 'h1', 'MAC': '00:00:00:00:00:01',
+                   'id':'00:00:00:00:00:01/-1' } , 'host2':
+                 { 'name': 'h9', 'MAC': '00:00:00:00:00:09',
+                   'id':'00:00:00:00:00:09/-1'}, 'link': { 'switch1': 's5',
+                   'switch2': 's2', 'num':'18' } }
+        dualStack1 = { 'name': 'DUALSTACK1', 'host1':
+                 { 'name': 'h3', 'MAC': '00:00:00:00:00:03',
+                   'id':'00:00:00:00:00:03/-1' } , 'host2':
+                 { 'name': 'h11', 'MAC': '00:00:00:00:00:0B',
+                   'id':'00:00:00:00:00:0B/-1'}, 'link': { 'switch1': 's5',
+                   'switch2': 's2', 'num':'18' } }
+        items.append( ipv4 )
+        items.append( dualStack1 )
+        # Global variables
+        
+        main.log.case( "Add host intents between 2 host" )
+        
+        for item in items:
+            stepResult = main.TRUE
+            itemName = item[ 'name' ]
+            h1Name = item[ 'host1' ][ 'name' ]
+            h2Name = item[ 'host2' ][ 'name' ]
+            h1Mac = item[ 'host1' ][ 'MAC' ]
+            h2Mac = item[ 'host2' ][ 'MAC' ]
+            h1Id = item[ 'host1' ][ 'id']
+            h2Id = item[ 'host2' ][ 'id']
+            # Link down/up for rerouting
+            sw1 = item[ 'link' ][ 'switch1' ]
+            sw2 = item[ 'link' ][ 'switch2' ]
+            remLink = item[ 'link' ][ 'num' ]
+            pingResult = main.TRUE
+            statusResult = main.TRUE
+            linkDownResult = main.TRUE
+            linkUpResult = main.TRUE
+            intentsId = []
+            main.step( itemName + ": Add host intents between " + h1Name
+                        + " and " + h2Name )
+            main.log.info( itemName + ": Discover host using arping" )
+            main.Mininet1.arping( host=h1Name )
+            main.Mininet1.arping( host=h2Name )
+            host1 = main.CLIs[ 0 ].getHost( mac=h1Mac )
+            host2 = main.CLIs[ 0 ].getHost( mac=h2Mac )
+            print host1
+            print host2
+            # Adding intents
+            main.log.info( itemName + ": Adding host intents" )
+            intent1 = main.CLIs[ 0 ].addHostIntent( hostIdOne=h1Id,
+                                                   hostIdTwo=h2Id )
+            intentsId.append( intent1 )
+            time.sleep( 5 )
+            intent2 = main.CLIs[ 0 ].addHostIntent( hostIdOne=h2Id,
+                                                   hostIdTwo=h1Id )
+            intentsId.append( intent2 )
+            # Checking intents
+            main.log.info( itemName + ": Check host intents state" )
+            time.sleep( 30 )
+            for i in range( main.numCtrls ):
+                intentResult = main.CLIs[ i ].checkIntentState(
+                                                          intentsId=intentsId )
+            if not intentResult:
+                main.log.info( itemName +  ": Check host intents state" +
+                               " again")
+                for i in range( main.numCtrls ):
+                    intentResult = main.CLIs[ i ].checkIntentState(
+                                                          intentsId=intentsId )
+            # Ping hosts
+            time.sleep( 10 )
+            main.log.info( itemName + ": Ping " + h1Name + " and " +
+                           h2Name )
+            pingResult1 = main.Mininet1.pingHost( src=h1Name , target=h2Name )
+            if not pingResult1:
+                main.log.info( itemName + ": " + h1Name + " cannot ping "
+                               + h2Name )
+            pingResult2 = main.Mininet1.pingHost( src=h2Name , target=h1Name )
+            if not pingResult2:
+                main.log.info( itemName + ": " + h2Name + " cannot ping "
+                               + h1Name )
+            pingResult = pingResult1 and pingResult2
+            if pingResult:
+                main.log.info( itemName + ": Successfully pinged " +
+                               "both hosts" )
+            else:
+                main.log.info( itemName + ": Failed to ping " +
+                               "both hosts" )
+            # Rerouting ( link down )
+            main.log.info( itemName + ": Bring link down between " +
+                           sw1 + " and " + sw2 )
+            main.Mininet1.link( end1=sw1,
+                                end2=sw2,
+                                option="down" )
+            time.sleep( 5 )
+
+            # Check onos topology
+            main.log.info( itemName + ": Checking ONOS topology " )
+
+            for i in range( main.numCtrls ):
+                topologyResult = main.CLIs[ i ].topology()
+                linkDownResult = main.ONOSbench.checkStatus( topologyResult,
+                                                           main.numSwitch,
+                                                           remLink )\
+                               and linkDownResult
+            if not linkDownResult:
+                main.log.info( itemName + ": Topology mismatch" )
+            else:
+                main.log.info( itemName + ": Topology match" )
+
+            # Ping hosts
+            main.log.info( itemName + ": Ping " + h1Name + " and " +
+                           h2Name )
+            pingResult1 = main.Mininet1.pingHost( src=h1Name , target=h2Name )
+            if not pingResult1:
+                main.log.info( itemName + ": " + h1Name + " cannot ping "
+                               + h2Name )
+            pingResult2 = main.Mininet1.pingHost( src=h2Name , target=h1Name )
+            if not pingResult2:
+                main.log.info( itemName + ": " + h2Name + " cannot ping "
+                               + h1Name )
+            pingResult = pingResult1 and pingResult2
+            if pingResult:
+                main.log.info( itemName + ": Successfully pinged " +
+                               "both hosts" )
+            else:
+                main.log.info( itemName + ": Failed to ping " +
+                               "both hosts" )
+            # link up
+            main.log.info( itemName + ": Bring link up between " +
+                           sw1 + " and " + sw2 )
+            main.Mininet1.link( end1=sw1,
+                                end2=sw2,
+                                option="up" )
+            time.sleep( 5 )
+
+            # Check onos topology
+            main.log.info( itemName + ": Checking ONOS topology " )
+            for i in range( main.numCtrls ):
+                topologyResult = main.CLIs[ i ].topology()
+                linkUpResult = main.ONOSbench.checkStatus( topologyResult,
+                                                           main.numSwitch,
+                                                           main.numLinks )\
+                               and linkUpResult
+            if not linkUpResult:
+                main.log.info( itemName + ": Topology mismatch" )
+            else:
+                main.log.info( itemName + ": Topology match" )
+
+            # Ping hosts
+            main.log.info( itemName + ": Ping " + h1Name + " and " +
+                           h2Name )
+            pingResult1 = main.Mininet1.pingHost( src=h1Name , target=h2Name )
+            if not pingResult1:
+                main.log.info( itemName + ": " + h1Name + " cannot ping "
+                               + h2Name )
+            pingResult2 = main.Mininet1.pingHost( src=h2Name , target=h1Name )
+            if not pingResult2:
+                main.log.info( itemName + ": " + h2Name + " cannot ping "
+                               + h1Name )
+            pingResult = pingResult1 and pingResult2
+            if pingResult:
+                main.log.info( itemName + ": Successfully pinged " +
+                               "both hosts" )
+            else:
+                main.log.info( itemName + ": Failed to ping " +
+                               "both hosts" )
+
+            # Remove intents
+            for intent in intentsId:
+                main.CLIs[ 0 ].removeIntent( intentId=intent, purge=True )
+
+            print main.CLIs[ 0 ].intents()
+            stepResult = pingResult and linkDownResult and linkUpResult \
+                         and intentResult
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=stepResult,
+                                     onpass=itemName +
+                                            ": host intent successful",
+                                     onfail=itemName +
+                                            ": Add host intent failed" )
+    def CASE1002( self, main ):
+        """
+            Add point intents between 2 hosts:
+                - Get device ids
+                - Add point intents
+                - Check intents
+                - Check flows
+                - Ping hosts
+                - Reroute
+                    - Link down
+                    - Ping hosts
+                    - Link up
+                    - Ping hosts
+                - Remove intents
+        """
+
+    def CASE1003( self, main ):
+        """
+            Add single point to multi point intents
+                - Get device ids
+                - Add single point to multi point intents
+                - Check intents
+                - Check flows
+                - Ping hosts
+                - Reroute
+                    - Link down
+                    - Ping hosts
+                    - Link up
+                    - Ping hosts
+                - Remove intents
+        """
+
+    def CASE1004( self, main ):
+        """
+            Add multi point to single point intents
+                - Get device ids
+                - Add multi point to single point intents
+                - Check intents
+                - Check flows
+                - Ping hosts
+                - Reroute
+                    - Link down
+                    - Ping hosts
+                    - Link up
+                    - Ping hosts
+                - Remove intents
+        """
diff --git a/TestON/tests/FuncIntent/FuncIntent.topo b/TestON/tests/FuncIntent/FuncIntent.topo
new file mode 100755
index 0000000..e85b397
--- /dev/null
+++ b/TestON/tests/FuncIntent/FuncIntent.topo
@@ -0,0 +1,52 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOSbench>
+            <host>10.128.5.55</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+                <home>~/ONOS</home>
+            </COMPONENTS>
+        </ONOSbench>
+
+        <ONOScli1>
+            <host>10.128.5.55</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli1>
+
+        <ONOScli2>
+            <host>10.128.5.55</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli2>
+
+         <ONOScli3>
+            <host>10.128.5.55</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOScli3>
+
+        <Mininet1>
+            <host>10.128.10.20</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>MininetCliDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </Mininet1>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.params b/TestON/tests/IntentEventTP/IntentEventTP.params
index fb2e625..ddda23e 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.params
+++ b/TestON/tests/IntentEventTP/IntentEventTP.params
@@ -1,19 +1,20 @@
 <PARAMS>
 
-    <testcases>1,2,1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2,1,2,1,2,1,2</testcases>
     
+    <isOnBaremetal>True</isOnBaremetal>
     <debugMode></debugMode>  #nothing means false 
 
     <ENV>
     <cellName>intentTP</cellName>
-    <cellApps>null,intentperf</cellApps>
+    <cellApps>drivers,null,intentperf</cellApps>
     </ENV>
 
-    <SCALE>1,3,5,7</SCALE>
+    <SCALE>1,3,3,5,5,7,7</SCALE>
     <availableNodes>7</availableNodes>
 
     <GIT>
-        <autopull>off</autopull>
+        <autopull>on</autopull>
         <checkout>master</checkout>
     </GIT>
 
@@ -32,32 +33,33 @@
         <ip4>10.128.5.54</ip4>
         <port4>6633</port4>
 
-        <ip5>10.128.5.65</ip5>
+        <ip5>10.128.5.55</ip5>
         <port5>6633</port5>
 
-        <ip6>10.128.5.66</ip6>
+        <ip6>10.128.5.56</ip6>
         <port6>6633</port6>
 
-         <ip7>10.128.5.67</ip7>
+         <ip7>10.128.5.57</ip7>
         <port7>6633</port7> 
     </CTRL>
 
-    <MN><ip1>10.128.5.55</ip1></MN>
+    <MN><ip1>10.128.5.50</ip1></MN>
 
     <BENCH>
         <user>admin</user>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.128.5.50</ip1>
     </BENCH>
 
     <TEST> 
         <loadFrom>1,1,1,1,1,1,1</loadFrom>                                     #generate load on server, 1 = generator on 
         <numSwitches>10,10,10,10,10,10,10</numSwitches>
         <skipCleanInstall>yes</skipCleanInstall>
-        <duration>30</duration>
+        <duration>400</duration>
         <log_interval>20</log_interval>
-        <numKeys>5000</numKeys>
+        <numKeys>40000</numKeys>
         <cyclePeriod>1000</cyclePeriod>
         <neighbors>0,a</neighbors>           #a == all nodes (-1)
+        <flowRuleBUEnabled>false</flowRuleBUEnabled>
     </TEST>
 
     <METRICS>
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.py b/TestON/tests/IntentEventTP/IntentEventTP.py
index 0e8af7d..99332f7 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.py
+++ b/TestON/tests/IntentEventTP/IntentEventTP.py
@@ -16,6 +16,8 @@
     def CASE1( self, main ):           
                                         
         import time                     
+        import os.path
+
         global init       
         try: 
             if type(init) is not bool: 
@@ -35,7 +37,13 @@
         skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
         cellName = main.params[ 'ENV' ][ 'cellName' ]        
         numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
+        flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
+        onBaremetal = main.params['isOnBaremetal']
+        homeDir = os.path.expanduser('~')
 
+        main.exceptions = [0]*11
+        main.warnings = [0]*11
+        main.errors = [0]*11
 
         # -- INIT SECTION, ONLY RUNS ONCE -- # 
         if init == False: 
@@ -44,7 +52,7 @@
             global ONOSIp                   #list of ONOS IP addresses
             global scale 
             global commit
-            
+
             clusterCount = 0
             ONOSIp = [ 0 ]
             scale = (main.params[ 'SCALE' ]).split(",")            
@@ -109,7 +117,24 @@
             myDistribution.append(numSwitches[node-1])
 
         #main.ONOSbench.createLinkGraphFile( BENCHIp,cellIp,myDistribution)
-       
+
+        if onBaremetal == "True":
+            filename = "/onos/tools/package/bin/onos-service"
+            serviceConfig = open(homeDir + filename, 'w+')
+            serviceConfig.write("#!/bin/bash\n ")
+            serviceConfig.write("#------------------------------------- \n ")
+            serviceConfig.write("# Starts ONOS Apache Karaf container\n ")
+            serviceConfig.write("#------------------------------------- \n ")
+            serviceConfig.write("#export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}\n ")
+            serviceConfig.write("""export JAVA_OPTS="${JAVA_OPTS:--Xms8G -Xmx8G}" \n """)
+            serviceConfig.write("")
+            serviceConfig.write("ONOS_HOME=/opt/onos \n ")
+            serviceConfig.write("")
+            serviceConfig.write("[ -d $ONOS_HOME ] && cd $ONOS_HOME || ONOS_HOME=$(dirname $0)/..\n")
+            serviceConfig.write("""${ONOS_HOME}/apache-karaf-$KARAF_VERSION/bin/karaf "$@" \n """)
+            serviceConfig.close()
+            main.log.info("Set /onos/tools/package/bin/onos-service with 8G Xms/Xmx Options.")
+      
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()  
 
@@ -133,17 +158,39 @@
 
         time.sleep(20)
 
-        #main.ONOSbench.handle.sendline("""onos $OC1 "cfg setorg.onosproject.provider.nil.NullProviders enabled true" """)
-        #main.ONOSbench.handle.expect(":~")
-        #print main.ONOSbench.handle.before
+
+        while True: 
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount """ + str(clusterCount*10) + """ " """)
+            main.ONOSbench.handle.expect(":~")
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
+            main.ONOSbench.handle.expect(":~")
+            if ("value=" + str(clusterCount*10)) in main.ONOSbench.handle.before:
+                main.log.info("Device count set")
+                main.log.info("before" + main.ONOSbench.handle.before)
+                break
+            time.sleep(10)
+            main.log.info("cfg set failure, retrying")
+            main.log.info("before" + main.ONOSbench.handle.before)
         
-        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount """ + str(clusterCount*10) + """ " """)
+        while True:
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
+            main.ONOSbench.handle.expect(":~")
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
+            main.ONOSbench.handle.expect(":~")
+            if ("value=linear") in main.ONOSbench.handle.before:
+                main.log.info("Device count set")
+                main.log.info("before" + main.ONOSbench.handle.before)
+                break
+            time.sleep(10)
+            main.log.info("cfg set failure, retrying")
+            main.log.info("before" + main.ONOSbench.handle.before)
+
+        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBU + """" """)
         main.ONOSbench.handle.expect(":~")
-        print main.ONOSbench.handle.before
-        time.sleep(10)
-        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
+        main.ONOSbench.handle.sendline("""onos $OC1 "cfg get" """)
         main.ONOSbench.handle.expect(":~")
-        print main.ONOSbench.handle.before
+        main.log.info(main.ONOSbench.handle.before)
+
         time.sleep(10)
         main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
         main.ONOSbench.handle.expect(":~")
@@ -159,6 +206,7 @@
         while True:
             main.ONOSbench.handle.sendline("onos $OC1 summary")
             main.ONOSbench.handle.expect(":~")
+            main.log.info("before" + main.ONOSbench.handle.before)
             clusterCheck = ((main.ONOSbench.handle.before).splitlines())[3]
             print("\nBefore: " + str(clusterCheck))
             if ("SCC(s)=1,") in clusterCheck:
@@ -174,7 +222,7 @@
                     break
             lastOutput = clusterCheck
             time.sleep(5)
-
+        main.ONOSbench.onosErrorLog(ONOSIp[1])
 
     def CASE2( self, main ): 
         import time
@@ -182,6 +230,26 @@
         import string
         import csv
         import numpy
+        import os.path
+
+        global currentNeighbors
+        neighbors = []
+
+        try:
+            currentNeighbors
+        except:
+            currentNeighbors = "0"
+            neighbors = ['0']
+        else:
+            if currentNeighbors == "r":      #reset
+                currentNeighbors = "a"
+                neighbors = ['0']
+            else:
+                currentNeighbors = "r" 
+                neighbors = ['a']
+
+        if clusterCount == 1:
+            currentNeighbors = "r"
 
         main.log.info("Cluster Count = " + str(clusterCount))
 
@@ -193,7 +261,7 @@
         debug = main.params[ 'debugMode' ]
         numKeys = main.params[ 'TEST' ][ 'numKeys' ]
         cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
-        neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",") 
+        #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",") 
         metricList = [intentsRate, intentsWithdrawn, intentsFailed]
 
         for n in range(0, len(neighbors)): 
@@ -265,7 +333,7 @@
                 clusterTotal = str(numpy.sum(groupResult))
                 main.log.report("Results from this round of polling: " + str(groupResult))
                 main.log.report("Cluster Total: " + clusterTotal + "\n")
-            
+
             cmd = "onos $OC1 intent-perf-stop"
             main.ONOSbench.handle.sendline(cmd)
             main.ONOSbench.handle.expect(":~")
@@ -284,6 +352,7 @@
                 resultsDB.write(resultString)
             
             resultsDB.close() 
-
+            
+            main.ONOSbench.onosErrorLog(ONOSIp[1])
                         
 
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.topo b/TestON/tests/IntentEventTP/IntentEventTP.topo
index 30a1467..763a4d6 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.topo
+++ b/TestON/tests/IntentEventTP/IntentEventTP.topo
@@ -3,7 +3,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -12,7 +12,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
         </ONOS2cli>
 
         <ONOS3cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
         </ONOS3cli>
 
         <ONOS4cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
         </ONOS4cli>
 
         <ONOS5cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
         </ONOS5cli>
 
         <ONOS6cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
         </ONOS6cli>
 
         <ONOS7cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -112,7 +112,7 @@
 
     
         <ONOS5>
-            <host>10.128.5.65</host>
+            <host>10.128.5.55</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -121,7 +121,7 @@
         </ONOS5>
 
         <ONOS6>
-            <host>10.128.5.66</host>
+            <host>10.128.5.56</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -130,7 +130,7 @@
         </ONOS6>
 
         <ONOS7>
-            <host>10.128.5.67</host>
+            <host>10.128.5.57</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -141,6 +141,4 @@
     </COMPONENT>
 
 </TOPOLOGY>
-
-
-    
+ 
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
index 3185cc2..2b76662 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
@@ -1,22 +1,22 @@
 <PARAMS>
 
-    <testcases>1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2</testcases>
 
-    <SCALE>3,5,7</SCALE>
+    <SCALE>1,3,5,7</SCALE>
     <availableNodes>7</availableNodes>
  
     <ENV>
         <cellName>IntentInstallWithdrawCell</cellName>
-        <cellApps>null</cellApps>
+        <cellApps>drivers,null</cellApps>
     </ENV>
 
     <TEST>
         <skipCleanInstall>yes</skipCleanInstall>
         <switchCount>7</switchCount>
-        <warmUp>4</warmUp>
-        <sampleSize>10</sampleSize>                     
+        <warmUp>10</warmUp>
+        <sampleSize>20</sampleSize>                     
         <wait></wait>
-        <intents>1000</intents>                       #list format, will be split on ','
+        <intents>1,100,1000</intents>                       #list format, will be split on ','
         <debug>True</debug>                                        #"True" for true
     </TEST>
 
@@ -28,36 +28,36 @@
     <CTRL>
         <USER>admin</USER>
         
-        <ip1>10.128.5.51</ip1>
+        <ip1>10.254.1.201</ip1>
         <port1>6633</port1>
         
-        <ip2>10.128.5.52</ip2>
+        <ip2>10.254.1.202</ip2>
         <port2>6633</port2>
         
-        <ip3>10.128.5.53</ip3>
+        <ip3>10.254.1.203</ip3>
         <port3>6633</port3>
         
-        <ip4>10.128.5.54</ip4>
+        <ip4>10.254.1.204</ip4>
         <port4>6633</port4>
         
-        <ip5>10.128.5.65</ip5>
+        <ip5>10.254.1.205</ip5>
         <port5>6633</port5>
         
-        <ip6>10.128.5.66</ip6>
+        <ip6>10.254.1.206</ip6>
         <port6>6633</port6> 
        
-        <ip7>10.128.5.67</ip7>
+        <ip7>10.254.1.207</ip7>
         <port7>6633</port7>
 
     </CTRL>
 
     <MN>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.254.1.200</ip1>
     </MN>
 
     <BENCH>
         <user>admin</user>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.254.1.200</ip1>
     </BENCH>
 
     <JSON>
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
index be8f38c..83dac2a 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
@@ -252,6 +252,6 @@
             resultString += str(numpy.std(installed)) + ","
             resultString += str(numpy.mean(withdrawn)) + ","
             resultString += str(numpy.std(withdrawn)) + "\n"
-            resultsDB = open("IntentInstallWithdrawLatDB", "w+")
+            resultsDB = open("IntentInstallWithdrawLatDB", "a")
             resultsDB.write(resultString)
             resultsDB.close()
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
index 30a1467..0e45e0f 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
@@ -3,7 +3,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -12,7 +12,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
         </ONOS2cli>
 
         <ONOS3cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
         </ONOS3cli>
 
         <ONOS4cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
         </ONOS4cli>
 
         <ONOS5cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
         </ONOS5cli>
 
         <ONOS6cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
         </ONOS6cli>
 
         <ONOS7cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -75,7 +75,7 @@
         </ONOS7cli>
 
         <ONOS1>
-            <host>10.128.5.51</host>
+            <host>10.254.1.201</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -84,7 +84,7 @@
         </ONOS1>
 
         <ONOS2>
-            <host>10.128.5.52</host>
+            <host>10.254.1.202</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -93,7 +93,7 @@
         </ONOS2>
 
         <ONOS3>
-            <host>10.128.5.53</host>
+            <host>10.254.1.203</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -102,7 +102,7 @@
         </ONOS3>
 
         <ONOS4>
-            <host>10.128.5.54</host>
+            <host>10.254.1.204</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -112,7 +112,7 @@
 
     
         <ONOS5>
-            <host>10.128.5.65</host>
+            <host>10.254.1.205</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -121,7 +121,7 @@
         </ONOS5>
 
         <ONOS6>
-            <host>10.128.5.66</host>
+            <host>10.254.1.206</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -130,7 +130,7 @@
         </ONOS6>
 
         <ONOS7>
-            <host>10.128.5.67</host>
+            <host>10.254.1.207</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.params b/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
index 27af1f2..8e7a6a4 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
@@ -1,27 +1,28 @@
 <PARAMS>
 
-    <testcases>1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2</testcases>
 
-    <SCALE>1</SCALE>
+    <SCALE>1,3,5,7</SCALE>
     <availableNodes>7</availableNodes>
  
     <ENV>
         <cellName>intentRerouteCell</cellName>
-        <cellApps>null,intentperf,metrics</cellApps>
-        <scale1switches>8</scale1switches>
-        <scale3switches>2,2,3</scale3switches>
-        <scale5switches>2,2,1,1,1</scale5switches>
-        <scale7switches>1,1,1,1,1,1,1</scale7switches>
-    
+        <cellApps>drivers,null,intentperf,metrics</cellApps>
     </ENV>
 
     <TEST>
         <skipCleanInstall>yes</skipCleanInstall>
-        <warmUp>1</warmUp>
-        <sampleSize>2</sampleSize>                     
+        <warmUp>5</warmUp>
+        <sampleSize>10</sampleSize>                     
         <wait></wait>
-        <intents>1,10</intents>                       #list format, will be split on ','
-        <debug>True</debug>                          
+        <intents>1,100,1000</intents>                       #list format, will be split on ','
+        <debug>True</debug>     
+       
+        <s1>1,1,1,1,1,1,1,1</s1>
+        <s3>2,2,1,1,3,3,3,1</s3>
+        <s5>2,2,1,1,3,4,5,3</s5>
+        <s7>2,3,1,1,5,6,7,4</s7>
+
     </TEST>
 
     <METRICS>
@@ -40,36 +41,36 @@
     <CTRL>
         <USER>admin</USER>
         
-        <ip1>10.128.5.51</ip1>
+        <ip1>10.254.1.201</ip1>
         <port1>6633</port1>
         
-        <ip2>10.128.5.52</ip2>
+        <ip2>10.254.1.202</ip2>
         <port2>6633</port2>
         
-        <ip3>10.128.5.53</ip3>
+        <ip3>10.254.1.203</ip3>
         <port3>6633</port3>
         
-        <ip4>10.128.5.54</ip4>
+        <ip4>10.254.1.204</ip4>
         <port4>6633</port4>
         
-        <ip5>10.128.5.65</ip5>
+        <ip5>10.254.1.205</ip5>
         <port5>6633</port5>
         
-        <ip6>10.128.5.66</ip6>
+        <ip6>10.254.1.206</ip6>
         <port6>6633</port6> 
        
-        <ip7>10.128.5.67</ip7>
+        <ip7>10.254.1.207</ip7>
         <port7>6633</port7>
 
     </CTRL>
 
     <MN>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.254.1.200</ip1>
     </MN>
 
     <BENCH>
         <user>admin</user>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.254.1.200</ip1>
     </BENCH>
 
     <JSON>
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.py b/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
index 84d5c03..f824a5c 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
@@ -79,9 +79,6 @@
         clusterCount = int(scale[0])
         scale.remove(scale[0])       
        
-        switchParams = ("scale" + str(clusterCount) + "switches")
-        switchCount = (main.params[ 'ENV' ][ switchParams ]).split(",")
-
         #kill off all onos processes 
         main.log.step("Safety check, killing all ONOS processes")
         main.log.step("before initiating enviornment setup")
@@ -105,15 +102,6 @@
         main.step( "Set Cell" )
         main.ONOSbench.setCell(cellName)
         
-        if clusterCount == 1:
-            temp = "one"
-        if clusterCount == 3:
-            temp = "three"
-        if clusterCount == 5:
-            temp = "five"
-        if clusterCount == 7:
-            temp = "seven"
-
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()  
 
@@ -134,7 +122,10 @@
             if not isup:
                 main.log.report( "ONOS " + str(node) + " didn't start!" )
         main.log.info("Startup sequence complete")
-     
+    
+        deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
+        print("Device mastership list: " + str(deviceMastership))
+
         main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount 8 " """)
         main.ONOSbench.handle.expect(":~")
         print repr(main.ONOSbench.handle.before)
@@ -143,53 +134,49 @@
         main.ONOSbench.handle.expect(":~")
         print repr(main.ONOSbench.handle.before)
         time.sleep(3)
-        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled true" """)
-        main.ONOSbench.handle.expect(":~")
-        print repr(main.ONOSbench.handle.before) 
 
-        while True: 
+        time.sleep(10)
+
+        once = True
+        for attempt in range(0,10): 
             main.ONOSbench.handle.sendline("onos $OC1 summary")
             main.ONOSbench.handle.expect(":~")
             x = main.ONOSbench.handle.before
-            if "devices=8" in x:
+            main.log.info("Initial setup. ONOS summary response: \n" + x + "\n")
+
+            if "devices=8" in x and "links=16," in x: 
                 break
-            else:   
+            else:                
+                if once == False:
+                    main.log.error("Null provider start failed, retrying..")
+                once = False
                 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled false" """)
                 main.ONOSbench.handle.expect(":~")
                 time.sleep(3)
                 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled true" """)
                 main.ONOSbench.handle.expect(":~")
-                main.log.error("Null provider start failed, retrying..") 
-                time.sleep(8)
+                time.sleep(8) 
 
-        main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
-        main.ONOSbench.handle.expect(":~")
-        print main.ONOSbench.handle.before
-        time.sleep(10)
-        main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
-        main.ONOSbench.handle.expect(":~")
-        
-        temp = 1 
-        for node in range(1, clusterCount + 1): 
-            for switch in range (0, int(switchCount[node-1])): 
-                cmd = ("""onos $OC1 "device-role null:000000000000000""" + str(temp) + " " + ONOSIp[node] + """ master" """)
+        index = 1
+        for node in deviceMastership:
+            for attempt in range(0,10):
+                cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)]  + """ master" """)
+                main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
                 main.ONOSbench.handle.sendline(cmd)
-                main.log.info( cmd ) 
                 main.ONOSbench.handle.expect(":~")
-                temp += 1
+                time.sleep(4)
+                
+                cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index)) 
+                main.log.info(cmd) 
+                main.ONOSbench.handle.sendline(cmd)
+                main.ONOSbench.handle.expect(":~")
+                check = main.ONOSbench.handle.before
+                main.log.info("CHECK:\n" + check)
+                if ("master=" + ONOSIp[int(node)]) in check:
+                    break
+            index += 1
 
 
-        #    cmd = ( """onos $OC1 "device-role null:0000000000000008 10.128.5.52 master" """)
-        #if clusterCount == 7: 
-        #    cmd = ( """onos $OC1 "device-role null:0000000000000008 10.128.5.53 master" """)
-            
-        #main.ONOSbench.handle.sendline(cmd)
-        #main.log.info( cmd )
-        #main.ONOSbench.handle.expect(":~")
-
-        #print "sleeping"
-        #time.sleep(120)
-
     def CASE2( self, main ):
          
         import time
@@ -226,7 +213,7 @@
             debug = False
 
         ingress = "null:0000000000000001"
-        egress = "null:0000000000000008"
+        egress = "null:0000000000000007"
 
         for intents in intentsList:
             main.log.report("Intent Batch size: " + str(intents) + "\n      ")
@@ -236,7 +223,7 @@
                 if run > warmUp:
                     main.log.info("Starting test iteration " + str(run-warmUp))
 
-                cmd = """onos $OC1 push-test-intents -i" """
+                cmd = """onos $OC1 "push-test-intents -i """
                 cmd += ingress + "/0 "
                 cmd += egress + "/0 "
                 cmd += str(intents) +""" 1" """
@@ -253,10 +240,10 @@
                     main.ONOSbench.handle.sendline("onos $OC1 summary")
                     main.ONOSbench.handle.expect(":~")
                     linkCheck = main.ONOSbench.handle.before
-                    if ("flows=16,") in linkCheck:
+                    if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
                         break
                     if i == 39:
-                        main.log.error("Flow count incorrect, data invalid."+ linkCheck)
+                        main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
 
 
                 #collect timestamp from link cut
@@ -270,7 +257,8 @@
                     time.sleep(2)
                     main.ONOSbench.handle.expect(":~")
                     raw = main.ONOSbench.handle.before
-                    if "NullLinkProvider" in raw and "links=14" in raw:
+                    #if "NullLinkProvider" in raw and "links=14" in raw:
+                    if "links=14" in raw:
                         break
                     if i >= 9:
                         main.log.error("Expected output not being recieved... continuing")
@@ -292,22 +280,27 @@
                     main.ONOSbench.handle.sendline("onos $OC1 summary")
                     main.ONOSbench.handle.expect(":~")
                     linkCheck = main.ONOSbench.handle.before
-                    if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
+                    #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
+                    if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
                         break
                     if i == 39:
                         main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
+                
+                time.sleep(5) #trying to avoid negative values 
 
                 #intents events metrics installed timestamp
                 IEMtimestamps = [0]*(clusterCount + 1)
                 installedTemp = [0]*(clusterCount + 1)
                 for node in range(1, clusterCount +1):
-                    cmd = "onos $OC" + str(node) + " intents-events-metrics|grep Timestamp"
+                    cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
                     raw = ""
-                    while "Timestamp" not in raw:
+                    while "epoch)" not in raw:
                         main.ONOSbench.handle.sendline(cmd)
                         main.ONOSbench.handle.expect(":~")
                         raw = main.ONOSbench.handle.before
 
+                    print(raw)
+
                     intentsTimestamps = {}
                     rawTimestamps = raw.splitlines()
                     for line in rawTimestamps:
@@ -398,13 +391,16 @@
             average = numpy.average(latTemp)
             stdDev = numpy.std(latTemp)
 
+            average = numpy.multiply(average, 1000)
+            stdDev = numpy.multiply(stdDev, 1000)
+
             main.log.report("Scale: " + str(clusterCount) + "  \tIntent batch: " + str(intents))
             main.log.report("Latency average:................" + str(average))
             main.log.report("Latency standard deviation:....." + str(stdDev))
             main.log.report("Mode of last node to respond:..." + str(nodeMode))
             main.log.report("________________________________________________________")
 
-            resultsDB = open("IntentRerouteLatDB", "w+")
+            resultsDB = open("IntentRerouteLatDB", "a")
             resultsDB.write("'" + commit + "',") 
             resultsDB.write(str(clusterCount) + ",")
             resultsDB.write(str(intents) + ",")
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo b/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
index 30a1467..0e45e0f 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
@@ -3,7 +3,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -12,7 +12,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
         </ONOS2cli>
 
         <ONOS3cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
         </ONOS3cli>
 
         <ONOS4cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
         </ONOS4cli>
 
         <ONOS5cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
         </ONOS5cli>
 
         <ONOS6cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
         </ONOS6cli>
 
         <ONOS7cli>
-            <host>10.128.5.55</host>
+            <host>10.254.1.200</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -75,7 +75,7 @@
         </ONOS7cli>
 
         <ONOS1>
-            <host>10.128.5.51</host>
+            <host>10.254.1.201</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -84,7 +84,7 @@
         </ONOS1>
 
         <ONOS2>
-            <host>10.128.5.52</host>
+            <host>10.254.1.202</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -93,7 +93,7 @@
         </ONOS2>
 
         <ONOS3>
-            <host>10.128.5.53</host>
+            <host>10.254.1.203</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -102,7 +102,7 @@
         </ONOS3>
 
         <ONOS4>
-            <host>10.128.5.54</host>
+            <host>10.254.1.204</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -112,7 +112,7 @@
 
     
         <ONOS5>
-            <host>10.128.5.65</host>
+            <host>10.254.1.205</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -121,7 +121,7 @@
         </ONOS5>
 
         <ONOS6>
-            <host>10.128.5.66</host>
+            <host>10.254.1.206</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -130,7 +130,7 @@
         </ONOS6>
 
         <ONOS7>
-            <host>10.128.5.67</host>
+            <host>10.254.1.207</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
diff --git a/TestON/tests/flowTP1g/flowTP1g.params b/TestON/tests/flowTP1g/flowTP1g.params
index 0f08c9a..54320a7 100644
--- a/TestON/tests/flowTP1g/flowTP1g.params
+++ b/TestON/tests/flowTP1g/flowTP1g.params
@@ -1,26 +1,27 @@
 <PARAMS>
 
-    <testcases>1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2</testcases>
     
     <isOnBaremetal>False</isOnBaremetal>
-    <SCALE>3</SCALE>
+    <SCALE>1,3,3,5,5,7,7</SCALE>
     <availableNodes>7</availableNodes>
     
 
 
     <ENV>
         <cellName>flowTP</cellName>
-        <cellApps>null,demo</cellApps>
+        <cellApps>drivers,null,demo</cellApps>
     </ENV>
 
     <TEST>
+        <enableFlowRuleStoreBackup>true</enableFlowRuleStoreBackup>
         <skipCleanInstall>yes</skipCleanInstall> 
-        <warmUp>4</warmUp>                              #number of runs to warm up the system
-        <sampleSize>3</sampleSize>                     #number of runs to take data from
+        <warmUp>5</warmUp>                              #number of runs to warm up the system
+        <sampleSize>20</sampleSize>                     #number of runs to take data from
         <neighbors>0,a</neighbors>                      #list of number of neighbors, a = all 
-        <flows>3500</flows>
+        <flows>122500</flows>
         <switches>35</switches>
-
+        <cooldown>10</cooldown>
         <testCMD0>flow-tester.py -f</testCMD0>          #base command 
         <testCMD1> -n </testCMD1>                       #neighbors 
         
@@ -46,24 +47,24 @@
         <ip4>10.128.5.54</ip4>
         <port4>6633</port4>
         
-        <ip5>10.128.5.65</ip5>
+        <ip5>10.128.5.55</ip5>
         <port5>6633</port5>
         
-        <ip6>10.128.5.66</ip6>
+        <ip6>10.128.5.56</ip6>
         <port6>6633</port6> 
        
-        <ip7>10.128.5.67</ip7>
+        <ip7>10.128.5.57</ip7>
         <port7>6633</port7>
 
     </CTRL>
 
     <MN>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.128.5.59</ip1>
     </MN>
 
     <BENCH>
         <user>admin</user>
-        <ip1>10.128.5.55</ip1>
+        <ip1>10.128.5.50</ip1>
     </BENCH>
 
     <JSON>
diff --git a/TestON/tests/flowTP1g/flowTP1g.py b/TestON/tests/flowTP1g/flowTP1g.py
index 352341e..cbea039 100644
--- a/TestON/tests/flowTP1g/flowTP1g.py
+++ b/TestON/tests/flowTP1g/flowTP1g.py
@@ -34,9 +34,12 @@
         maxNodes = int(main.params[ 'availableNodes' ])
         skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
         cellName = main.params[ 'ENV' ][ 'cellName' ]       
- 
-        #----do i need this line?
-        #main.ONOSbench.handle.sendline("export TERM=vt100")
+    
+        main.log.info("==========DEBUG VERSION 3===========")
+
+        main.exceptions = [0]*11
+        main.warnings = [0]*11
+        main.errors = [0]*11
 
         # -- INIT SECTION, ONLY RUNS ONCE -- #
         if init == False:
@@ -132,7 +135,7 @@
             a(ONOSIp[node])
          
         main.log.info("Startup sequence complete")
-
+        main.ONOSbench.onosErrorLog(ONOSIp[1])
         
     def CASE2( self, main ):
         #
@@ -145,6 +148,17 @@
         import datetime
         import traceback
 
+        global currentNeighbors
+        try:
+            currentNeighbors
+        except:
+            currentNeighbors = "0"
+        else:
+            if currentNeighbors == "r":      #reset
+                currentNeighbors = "0"
+            else:
+                currentNeighbors = "a"
+
         testCMD = [ 0,0,0,0 ]
         warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
         sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
@@ -154,30 +168,33 @@
         testCMD[1] = main.params[ 'TEST' ][ 'testCMD1' ]
         maxNodes = main.params[ 'availableNodes' ]
         onBaremetal = main.params['isOnBaremetal']
-
+        cooldown = main.params[ 'TEST' ][ 'cooldown' ]
         cellName = main.params[ 'ENV' ][ 'cellName' ]
         BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
         BENCHUser = main.params[ 'BENCH' ][ 'user' ]
         MN1Ip = main.params[ 'MN' ][ 'ip1' ]
         maxNodes = int(main.params[ 'availableNodes' ])
         homeDir = os.path.expanduser('~')
-  
+        flowRuleBackup = str(main.params[ 'TEST' ][ 'enableFlowRuleStoreBackup' ])
+        main.log.info("Flow Rule Backup is set to:" + flowRuleBackup)
+
         servers = str(clusterCount) 
-        for i in range(0, len(neighborList)):
-            if neighborList[i] == 'a':
-                neighborList[i] = str(clusterCount - 1)  
         
         if clusterCount == 1: 
             neighborList = ['0']
+            currentNeighbors = "r"
+        else:
+            if currentNeighbors == "a":
+                neighborList = [str(clusterCount-1)]
+                currentNeighbors = "r"
+            else:
+                neighborList = ['0'] 
+        
         main.log.info("neightborlist: " + str(neighborList))
 
         ts = time.time()
         st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
 
-        #initialize log file, remove any previous data
-        resultsLog = open("flowTP1gDB","w+")
-        resultsLog.close()
-
         #write file to change mem limit to 32 gigs (BAREMETAL ONLY!)
         if onBaremetal == "true":
             filename = "/onos/tools/package/bin/onos-service"
@@ -208,25 +225,33 @@
                 ipCSV += main.params[ 'CTRL' ][ tempstr ] 
                 if i < int(maxNodes):
                     ipCSV +=","
-            
-            filename = "/onos/tools/package/etc/org.onosproject.provider.nil.link.impl.NullLinkProvider.cfg"
-            linkConfig = open(homeDir + filename,'w+')
-            linkConfig.write("# eventRate = 2000\n")
-            linkConfig.write("neighbors = "  + ipCSV)
-            main.log.info(" NullLinkProvider.cfg: " + ipCSV)
-            linkConfig.close()
-            
-            main.ONOSbench.handle.sendline("""onos $OC1 "cfg setorg.onosproject.provider.nil.NullProviders enabled true" """)
-            main.ONOSbench.handle.expect(":~")
+           
+
             main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount 35" """)
             main.ONOSbench.handle.expect(":~")
+            time.sleep(3)
             main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
             main.ONOSbench.handle.expect(":~")
+            time.sleep(3)
             main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
             main.ONOSbench.handle.expect(":~")
+            time.sleep(3)
             main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
             main.ONOSbench.handle.expect(":~")
-        
+            time.sleep(3)
+            main.log.info("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBackup + """" """)
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBackup + """" """)
+            main.ONOSbench.handle.expect(":~")
+       
+            main.ONOSbench.handle.sendline("onos $OC1 summary")
+            main.ONOSbench.handle.expect(":~")
+            check = main.ONOSbench.handle.before
+
+            main.ONOSbench.handle.sendline("""onos $OC1 "cfg get" """)
+            main.ONOSbench.handle.expect(":~")
+            check = main.ONOSbench.handle.before
+            main.log.info("\nStart up check: \n" + check + "\n") 
+
             #devide flows
             flows = int(main.params[ 'TEST' ][ 'flows' ])
             main.log.info("Flow Target  = " + str(flows))
@@ -243,35 +268,51 @@
             data = [[""]*int(servers)]*int(sampleSize)
             maxes = [""]*int(sampleSize)
 
-            for test in range(0, (warmUp + sampleSize)): 
-                flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
-                flowCMD += testCMD[0] + " " + str(flows) + " " + testCMD[1] 
-                flowCMD += " " + str(n) + " " + str(serverEnvVars)
-                print("\n")                    
-                main.log.info("COMMAND: " + flowCMD)
-                main.log.info("Executing command") 
-                
-                for i in range(0,15):
-                    main.ONOSbench.handle.sendline(flowCMD)
-                    time.sleep(1)
-                    main.ONOSbench.handle.expect(":~")
-                    rawResult = main.ONOSbench.handle.before
-                    if " -> " in rawResult:
-                        break
+            flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
+            flowCMD += testCMD[0] + " " + str(flows) + " " + testCMD[1]
+            flowCMD += " " + str(n) + " " + str(serverEnvVars) + "-j" 
 
+            main.log.info(flowCMD)
+            #time.sleep(60)
+            
+            for test in range(0, warmUp + sampleSize): 
+                if test < warmUp: 
+                    main.log.info("Warm up " + str(test + 1) + " of " + str(warmUp))
+                else: 
+                     main.log.info("====== Test run: " + str(test-warmUp+1) + " ======") 
+
+                main.ONOSbench.handle.sendline(flowCMD)
+                main.ONOSbench.handle.expect(":~")
+                rawResult = main.ONOSbench.handle.before
+                main.log.info("Raw results: \n" + rawResult + "\n")
+
+                if "failed" in rawResult: 
+                    main.log.report("FLOW_TESTER.PY FAILURE")
+                    main.log.report( " \n" + rawResult + " \n") 
+                    break
+            
+            ########################################################################################
                 result = [""]*(clusterCount)
                 rawResult = rawResult.splitlines()
-                print(rawResult)
+
                 for node in range(1, clusterCount + 1):        
                     for line in rawResult:
-                        if ONOSIp[node] in line and " -> " in line:
-                            myLine = line.split(" ")
-                            for word in myLine:
-                                if "ms" in word:
-                                    result[node-1] = int(word.replace("ms",""))
-                                    main.log.info("Parsed result: " + str(result[node-1]))
-                                    break
-                            break
+                        #print("line: " + line) 
+                        if ONOSIp[node] in line and "server" in line:
+                            temp = line.split(" ") 
+                            for word in temp:
+                                #print ("word: " + word) 
+                                if "elapsed" in repr(word): 
+                                    #print("in elapsed ==========")
+                                    index = temp.index(word) + 1
+                                    #print ("index: " + str(index)) 
+                                    #print ("temp[index]: " + temp[index])
+                                    myParsed = (temp[index]).replace(",","")
+                                    myParsed = myParsed.replace("}","")
+                                    myParsed = int(myParsed)
+                                    result[node-1] = myParsed
+                                    main.log.info( ONOSIp[node] + " : " + str(myParsed))
+                                    break 
 
                 if test >= warmUp:
                     for i in result: 
@@ -283,40 +324,27 @@
                     main.log.info("Data collection iteration: " + str(test-warmUp) + " of " + str(sampleSize))
                     main.log.info("Throughput time: " + str(maxes[test-warmUp]) + "(ms)")                
 
-                if test >= warmUp:
                     data[test-warmUp] = result
 
                 # wait for flows = 0 
-                removedFlows = False
-                repeat = 0
-                time.sleep(3)
-                while ( removedFlows == False and repeat <= 10 ):
-                    main.ONOSbench.handle.sendline("onos $OC1 summary| cut -d ' ' -f6")
-                    main.ONOSbench.handle.expect("~")
-                    before = main.ONOSbench.handle.before
-                    parseTest = before.splitlines()
-                    flowsummary = ""
-                    for line in parseTest:
-                        if "flow" in str(line):
-                            flowsummary = line 
-                            break
-                    currentflow = ""
-                    for word in flowsummary.split(" "): 
-                        if "flow" in str(word):
-                            currentflow = str(word)
-                    currentflow = currentflow.replace(",","")
-                    currentflow = currentflow.replace("\n","")
-                    #main.log.info(currentflow)
+                for checkCount in range(0,5): 
+                    time.sleep(10)
+                    main.ONOSbench.handle.sendline("onos $OC1 summary")
+                    main.ONOSbench.handle.expect(":~")
+                    flowCheck = main.ONOSbench.handle.before
+                    if "flows=0," in flowCheck: 
+                        main.log.info("Flows removed")
+                        break
+                    else: 
+                        for line in flowCheck.splitlines(): 
+                            if "flows=" in line: 
+                                main.log.info("Current Summary: " + line) 
+                    if checkCount == 2: 
+                        main.log.info("Flows are stuck, moving on ")
 
-                    zeroFlow = "flows=0"                
-                    if zeroFlow in before:
-                        removedFlows = True 
-                        main.log.info("\t Wait 5 sec of cool down...")
-                        time.sleep(5)
 
-                    time.sleep(5)
-                    repeat +=1
-     
+                time.sleep(5)
+                
             main.log.info("raw data: " + str(data))
             main.log.info("maxes:" + str(maxes))
 
@@ -361,12 +389,15 @@
             main.log.info("Standard deviation of throughput: " + str(stdTP) + " Kflows/second") 
 
             resultsLog = open("flowTP1gDB","a")
-            resultsLog.write("'" + commit + "',")
-            resultsLog.write("'1gig',")
-            resultsLog.write((main.params[ 'TEST' ][ 'flows' ]) + ",")
-            resultsLog.write(str(clusterCount)+ ",")
-            resultsLog.write(str(n) + ",")
-            resultsLog.write(str(avgTP) + "," + str(stdTP) + "\n")
+            resultString = ("'" + commit + "',")
+            resultString += ("'1gig',")
+            resultString += ((main.params[ 'TEST' ][ 'flows' ]) + ",")
+            resultString += (str(clusterCount) + ",")
+            resultString += (str(n) + ",")
+            resultString += (str(avgTP) + "," + str(stdTP) + "\n")
+            resultsLog.write(resultString) 
             resultsLog.close()
-
-             
+            
+            main.log.report("Result line to file: " + resultString)
+           
+        main.ONOSbench.onosErrorLog(ONOSIp[1]) 
diff --git a/TestON/tests/flowTP1g/flowTP1g.topo b/TestON/tests/flowTP1g/flowTP1g.topo
index 30a1467..763a4d6 100644
--- a/TestON/tests/flowTP1g/flowTP1g.topo
+++ b/TestON/tests/flowTP1g/flowTP1g.topo
@@ -3,7 +3,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosDriver</type>
@@ -12,7 +12,7 @@
         </ONOSbench>
 
         <ONOS1cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
         </ONOS1cli>
 
         <ONOS2cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
         </ONOS2cli>
 
         <ONOS3cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
         </ONOS3cli>
 
         <ONOS4cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
         </ONOS4cli>
 
         <ONOS5cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
         </ONOS5cli>
 
         <ONOS6cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
         </ONOS6cli>
 
         <ONOS7cli>
-            <host>10.128.5.55</host>
+            <host>10.128.5.50</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>OnosCliDriver</type>
@@ -112,7 +112,7 @@
 
     
         <ONOS5>
-            <host>10.128.5.65</host>
+            <host>10.128.5.55</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -121,7 +121,7 @@
         </ONOS5>
 
         <ONOS6>
-            <host>10.128.5.66</host>
+            <host>10.128.5.56</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -130,7 +130,7 @@
         </ONOS6>
 
         <ONOS7>
-            <host>10.128.5.67</host>
+            <host>10.128.5.57</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosDriver</type>
@@ -141,6 +141,4 @@
     </COMPONENT>
 
 </TOPOLOGY>
-
-
-    
+