Initial commit for new OnosCHO
diff --git a/TestON/tests/OnosCHO/OnosCHO.py b/TestON/tests/OnosCHO/OnosCHO.py
index eb71518..16f2b5d 100644
--- a/TestON/tests/OnosCHO/OnosCHO.py
+++ b/TestON/tests/OnosCHO/OnosCHO.py
@@ -24,7 +24,9 @@
         onos-wait-for-start
         """
         import time
-
+        
+        main.threadID = 0
+        main.pingTimeout = 300
         main.numCtrls = main.params[ 'CTRL' ][ 'numCtrl' ]
         main.ONOS1_ip = main.params[ 'CTRL' ][ 'ip1' ]
         main.ONOS2_ip = main.params[ 'CTRL' ][ 'ip2' ]
@@ -86,7 +88,7 @@
         uninstallResult = main.TRUE
         for i in range( 1, int( main.numCtrls ) + 1 ):
             ONOS_ip = main.params[ 'CTRL' ][ 'ip' + str( i ) ]
-            main.log.info( "Unintsalling package on ONOS Node IP: " + ONOS_ip )
+            main.log.info( "Uninstalling package on ONOS Node IP: " + ONOS_ip )
             u_result = main.ONOSbench.onosUninstall( ONOS_ip )
             utilities.assert_equals( expect=main.TRUE, actual=u_result,
                                      onpass="Test step PASS",
@@ -123,20 +125,42 @@
         karafTimeout = "3600000"
         # need to wait here for sometime. This will be removed once ONOS is
         # stable enough
-        time.sleep( 20 )
-        for i in range( 1, int( main.numCtrls ) + 1 ):
-            ONOS_ip = main.params[ 'CTRL' ][ 'ip' + str( i ) ]
-            ONOScli = 'ONOScli' + str( i )
-            main.log.info( "ONOS Node " + ONOS_ip + " cli start:" )
-            exec "startcli=main." + ONOScli + \
-                ".startOnosCli(ONOS_ip, karafTimeout=karafTimeout)"
-            utilities.assert_equals( expect=main.TRUE, actual=startcli,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            cliResult = ( cliResult and startcli )
+        time.sleep( 25 )
 
-        case1Result = ( cp_result and cell_result
-                        and packageResult and installResult and statusResult and cliResult )
+        main.log.step(" Start ONOS cli using thread ")
+        startCliResult  = main.FALSE
+        
+        CLI1 = (main.ONOScli1.startOnosCli,main.ONOS1_ip)
+        CLI2 = (main.ONOScli2.startOnosCli,main.ONOS2_ip)
+        CLI3 = (main.ONOScli3.startOnosCli,main.ONOS3_ip)
+        CLI4 = (main.ONOScli4.startOnosCli,main.ONOS4_ip)
+        CLI5 = (main.ONOScli5.startOnosCli,main.ONOS5_ip)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        pool = []
+        time1 = time.time()
+        for cli,ip in ONOSCLI:
+            t = main.Thread(target=cli,threadID=main.threadID,
+                    name="startOnosCli",args=[ip])
+            pool.append(t)
+            t.start()
+            main.threadID = main.threadID + 1
+            
+        results = []
+        for thread in pool:
+            thread.join()
+            results.append(thread.result)
+        time2 = time.time()
+        
+        if( all(result == main.TRUE for result in results) == False):
+                main.log.info("ONOS CLI did not start up properly")
+                main.cleanup()
+                main.exit()
+        else:
+            main.log.info("Successful CLI startup")
+            startCliResult = main.TRUE
+        case1Result = installResult and uninstallResult and statusResult and startCliResult
+        
+        main.log.info("Time for connecting to CLI: %2f seconds" %(time2-time1))
         utilities.assert_equals( expect=main.TRUE, actual=case1Result,
                                  onpass="Set up test environment PASS",
                                  onfail="Set up test environment FAIL" )
@@ -151,7 +175,7 @@
         main.numMNswitches = int ( main.params[ 'TOPO1' ][ 'numSwitches' ] )
         main.numMNlinks = int ( main.params[ 'TOPO1' ][ 'numLinks' ] )
         main.numMNhosts = int ( main.params[ 'TOPO1' ][ 'numHosts' ] )
-
+        main.pingTimeout = 60
         main.log.report(
             "Assign and Balance all Mininet switches across controllers" )
         main.log.report(
@@ -217,20 +241,19 @@
         main.deviceLinks = []
         main.deviceActiveLinksCount = []
         main.devicePortsEnabledCount = []
-
+        
         main.log.report(
             "Collect and Store topology details from ONOS before running any Tests" )
         main.log.report(
             "____________________________________________________________________" )
         main.case( "Collect and Store Topology Deatils from ONOS" )
-
         main.step( "Collect and store current number of switches and links" )
         topology_output = main.ONOScli1.topology()
         topology_result = main.ONOSbench.getTopology( topology_output )
         numOnosDevices = topology_result[ 'devices' ]
         numOnosLinks = topology_result[ 'links' ]
 
-        if ( ( main.numMNswitches == int(numOnosDevices) ) and ( main.numMNlinks == int(numOnosLinks) ) ):
+        if ( ( main.numMNswitches == int(numOnosDevices) ) and ( main.numMNlinks >= int(numOnosLinks) ) ):
             main.step( "Store Device DPIDs" )
             for i in range( 1, (main.numMNswitches+1) ):
                 main.deviceDPIDs.append( "of:00000000000000" + format( i, '02x' ) )
@@ -255,22 +278,55 @@
             print "Length of Links Store", len( main.deviceLinks )
 
             main.step( "Collect and store each Device ports enabled Count" )
-            for i in range( 1, ( main.numMNswitches + 1) ):
-                portResult = main.ONOScli1.getDevicePortsEnabledCount(
-                    "of:00000000000000" + format( i,'02x' ) )
-                portTemp = re.split( r'\t+', portResult )
-                portCount = portTemp[ 1 ].replace( "\r\r\n\x1b[32m", "" )
-                main.devicePortsEnabledCount.append( portCount )
+            
+            CLI1 = (main.ONOScli1)
+            CLI2 = (main.ONOScli2)
+            CLI3 = (main.ONOScli3)
+            CLI4 = (main.ONOScli4)
+            CLI5 = (main.ONOScli5)
+            ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+            time1 = time.time()
+            
+            for i in xrange(1,(main.numMNswitches + 1),5):
+                pool = []
+                for cli in ONOSCLI:
+                    dpid = "of:00000000000000" + format( i,'02x' )
+                    t = main.Thread(target = cli.getDevicePortsEnabledCount,threadID = main.threadID, name = "getDevicePortsEnabledCount",args = [dpid])
+                    t.start()
+                    pool.append(t)
+                    i = i + 1
+                    main.threadID = main.threadID + 1
+                for thread in pool:
+                    thread.join()
+                    portResult = thread.result
+                    portTemp = re.split( r'\t+', portResult )
+                    portCount = portTemp[ 1 ].replace( "\r\r\n\x1b[32m", "" )
+                    main.devicePortsEnabledCount.append( portCount )
             print "Device Enabled Port Counts Stored: \n", str( main.devicePortsEnabledCount )
+            time2 = time.time()
+            main.log.info("Time for counting enabled ports of the switches: %2f seconds" %(time2-time1))
 
             main.step( "Collect and store each Device active links Count" )
-            for i in range( 1, ( main.numMNswitches + 1) ):
-                linkCountResult = main.ONOScli1.getDeviceLinksActiveCount(
-                    "of:00000000000000" + format( i,'02x' ) )
-                linkCountTemp = re.split( r'\t+', linkCountResult )
-                linkCount = linkCountTemp[ 1 ].replace( "\r\r\n\x1b[32m", "" )
-                main.deviceActiveLinksCount.append( linkCount )
-            print "Device Active Links Count Stored: \n", str( main.deviceActiveLinksCount )
+            time1 = time.time()
+            
+            for i in xrange(1,(main.numMNswitches + 1),5):
+                pool = []
+                for cli in ONOSCLI:
+                    dpid = "of:00000000000000" + format( i,'02x' )
+                    t = main.Thread(target = cli.getDeviceLinksActiveCount,threadID = main.threadID, name = "getDevicePortsEnabledCount",args = [dpid])
+                    t.start()
+                    pool.append(t)
+                    i = i + 1
+                    main.threadID = main.threadID + 1
+                for thread in pool:
+                    thread.join()
+                    linkCountResult = thread.result
+                    linkCountTemp = re.split( r'\t+', linkCountResult )
+                    linkCount = linkCountTemp[ 1 ].replace( "\r\r\n\x1b[32m", "" )
+                    main.deviceActiveLinksCount.append( linkCount )
+                print "Device Active Links Count Stored: \n", str( main.deviceActiveLinksCount )
+            time2 = time.time()
+            main.log.info("Time for counting all enabled links of the switches: %2f seconds" %(time2-time1))
 
         else:
             main.log.info("Devices (expected): %s, Links (expected): %s" % 
@@ -278,9 +334,9 @@
             main.log.info("Devices (actual): %s, Links (actual): %s" %
                     ( numOnosDevices , numOnosLinks ) )
             main.log.info("Topology does not match, exiting CHO test...")
-            time.sleep(300)
-            #main.cleanup()
-            #main.exit()
+            #time.sleep(300)
+            main.cleanup()
+            main.exit()
 
         # just returning TRUE for now as this one just collects data
         case3Result = main.TRUE
@@ -295,27 +351,48 @@
         import re
         import copy
         import time
-
         main.log.report( "Enable Reactive forwarding and Verify ping all" )
         main.log.report( "______________________________________________" )
         main.case( "Enable Reactive forwarding and Verify ping all" )
         main.step( "Enable Reactive forwarding" )
         installResult = main.TRUE
-        for i in range( 1, int( main.numCtrls ) + 1 ):
-            onosFeature = 'onos-app-fwd'
-            ONOS_ip = main.params[ 'CTRL' ][ 'ip' + str( i ) ]
-            ONOScli = 'ONOScli' + str( i )
-            main.log.info( "Enabling Reactive mode on ONOS Node " + ONOS_ip )
-            exec "inResult=main." + ONOScli + ".featureInstall(onosFeature)"
-            time.sleep( 3 )
-            installResult = inResult and installResult
-
+        feature = "onos-app-fwd"
+        CLI1 = (main.ONOScli1.featureInstall,feature)
+        CLI2 = (main.ONOScli2.featureInstall,feature)
+        CLI3 = (main.ONOScli3.featureInstall,feature)
+        CLI4 = (main.ONOScli4.featureInstall,feature)
+        CLI5 = (main.ONOScli5.featureInstall,feature)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        pool = []
+        time1 = time.time()
+        for cli,feature in ONOSCLI:
+            t = main.Thread(target=cli,threadID=main.threadID,
+                    name="featureInstall",args=[feature])
+            pool.append(t)
+            t.start()
+            main.threadID = main.threadID + 1
+            
+        results = []
+        for thread in pool:
+            thread.join()
+            results.append(thread.result)
+        time2 = time.time()
+        
+        if( all(result == main.TRUE for result in results) == False):
+                main.log.info("Did not install onos-app-fwd feature properly")
+                main.cleanup()
+                main.exit()
+        else:
+            main.log.info("Successful feature:install onos-app-fwd")
+            installResult = main.TRUE
+        main.log.info("Time for feature:install onos-app-fwd: %2f seconds" %(time2-time1))
+        
         time.sleep( 5 )
 
         main.step( "Verify Pingall" )
         ping_result = main.FALSE
         time1 = time.time()
-        ping_result = main.Mininet1.pingall()
+        ping_result = main.Mininet1.pingall(timeout=main.pingTimeout)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -329,20 +406,41 @@
             main.log.report( "Pingall Test in Reactive mode failed" )
 
         main.step( "Disable Reactive forwarding" )
-        uninstallResult = main.TRUE
-        for i in range( 1, int( main.numCtrls ) + 1 ):
-            onosFeature = 'onos-app-fwd'
-            ONOS_ip = main.params[ 'CTRL' ][ 'ip' + str( i ) ]
-            ONOScli = 'ONOScli' + str( i )
-            main.log.info( "Disabling Reactive mode on ONOS Node " + ONOS_ip )
-            exec "unResult=main." + ONOScli + ".featureUninstall(onosFeature)"
-            uninstallResult = unResult and uninstallResult
+        uninstallResult = main.FALSE
+        CLI1 = (main.ONOScli1.featureUninstall,"onos-app-fwd")
+        CLI2 = (main.ONOScli2.featureUninstall,"onos-app-fwd")
+        CLI3 = (main.ONOScli3.featureUninstall,"onos-app-fwd")
+        CLI4 = (main.ONOScli4.featureUninstall,"onos-app-fwd")
+        CLI5 = (main.ONOScli5.featureUninstall,"onos-app-fwd")
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        pool = []
+        time1 = time.time()
+        for cli,feature in ONOSCLI:
+            t = main.Thread(target=cli,threadID=main.threadID,
+                    name="featureUninstall",args=[feature])
+            pool.append(t)
+            t.start()
+            main.threadID = main.threadID + 1
+            
+        results = []
+        for thread in pool:
+            thread.join()
+            results.append(thread.result)
+        time2 = time.time()
+        
+        if( all(result == main.TRUE for result in results) == False):
+                main.log.info("Did not uninstall onos-app-fwd feature properly")
+                main.cleanup()
+                main.exit()
+        else:
+            main.log.info("Successful feature:uninstall onos-app-fwd")
+            uninstallResult = main.TRUE
+        main.log.info("Time for feature:uninstall onos-app-fwd: %2f seconds" %(time2-time1))
 
         # Waiting for reative flows to be cleared.
-        time.sleep( 10 )
-
-        case3Result = installResult and ping_result and uninstallResult
-        utilities.assert_equals( expect=main.TRUE, actual=case3Result,
+        time.sleep( 5 )
+        case4Result =  installResult and uninstallResult and ping_result
+        utilities.assert_equals( expect=main.TRUE, actual=case4Result,
                                  onpass="Reactive Mode Pingall test PASS",
                                  onfail="Reactive Mode Pingall test FAIL" )
 
@@ -351,6 +449,7 @@
         Compare current ONOS topology with reference data
         """
         import re
+        
         devicesDPIDTemp = []
         hostMACsTemp = []
         deviceLinksTemp = []
@@ -363,6 +462,36 @@
         main.case( "Compare ONOS topology with reference data" )
 
         main.step( "Compare current Device ports enabled with reference" )
+
+        CLI1 = (main.ONOScli1)
+        CLI2 = (main.ONOScli2)
+        CLI3 = (main.ONOScli3)
+        CLI4 = (main.ONOScli4)
+        CLI5 = (main.ONOScli5)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        time1 = time.time()
+        
+        for i in xrange(1,(main.numMNswitches + 1),5):
+            pool = []
+            for cli in ONOSCLI:
+                dpid = "of:00000000000000" + format( i,'02x' )
+                t = main.Thread(target = cli.getDevicePortsEnabledCount,threadID = main.threadID, name = "getDevicePortsEnabledCount",args = [dpid])
+                t.start()
+                pool.append(t)
+                i = i + 1
+                main.threadID = main.threadID + 1
+            for thread in pool:
+                thread.join()
+                portResult = thread.result
+                portTemp = re.split( r'\t+', portResult )
+                portCount = portTemp[ 1 ].replace( "\r\r\n\x1b[32m", "" )
+                devicePortsEnabledCountTemp.append( portCount )
+        print "Device Enabled Port Counts Stored: \n", str( main.devicePortsEnabledCount )
+        time2 = time.time()
+        main.log.info("Time for counting enabled ports of the switches: %2f seconds" %(time2-time1))
+        
+        
+        """
         for i in range( 1, 26 ):
             portResult = main.ONOScli1.getDevicePortsEnabledCount(
                 "of:00000000000000" +
@@ -373,12 +502,14 @@
             portCount = portTemp[ 1 ].replace( "\r\r\n\x1b[32m", "" )
             devicePortsEnabledCountTemp.append( portCount )
             time.sleep( 2 )
+        """
         main.log.info (
             "Device Enabled ports EXPECTED: %s" % 
 	     str( main.devicePortsEnabledCount ) )
         main.log.info (
             "Device Enabled ports ACTUAL: %s" % 
             str( devicePortsEnabledCountTemp ) )
+        
         if ( cmp( main.devicePortsEnabledCount,
                   devicePortsEnabledCountTemp ) == 0 ):
             stepResult1 = main.TRUE
@@ -386,16 +517,25 @@
             stepResult1 = main.FALSE
 
         main.step( "Compare Device active links with reference" )
-        for i in range( 1, 26 ):
-            linkResult = main.ONOScli1.getDeviceLinksActiveCount(
-                "of:00000000000000" +
-                format(
-                    i,
-                    '02x' ) )
-            linkTemp = re.split( r'\t+', linkResult )
-            linkCount = linkTemp[ 1 ].replace( "\r\r\n\x1b[32m", "" )
-            deviceActiveLinksCountTemp.append( linkCount )
-            time.sleep( 3 )
+        time1 = time.time()
+        for i in xrange(1,(main.numMNswitches + 1),5):
+            pool = []
+            for cli in ONOSCLI:
+                dpid = "of:00000000000000" + format( i,'02x' )
+                t = main.Thread(target = cli.getDeviceLinksActiveCount,threadID = main.threadID, name = "getDevicePortsEnabledCount",args = [dpid])
+                t.start()
+                pool.append(t)
+                i = i + 1
+                main.threadID = main.threadID + 1
+            for thread in pool:
+                thread.join()
+                linkCountResult = thread.result
+                linkCountTemp = re.split( r'\t+', linkCountResult )
+                linkCount = linkCountTemp[ 1 ].replace( "\r\r\n\x1b[32m", "" )
+                deviceActiveLinksCountTemp.append( linkCount )
+            print "Device Active Links Count Stored: \n", str( main.deviceActiveLinksCount )
+        time2 = time.time()
+        main.log.info("Time for counting all enabled links of the switches: %2f seconds" %(time2-time1))
         main.log.info (
             "Device Active links EXPECTED: %s" %
               str( main.deviceActiveLinksCount ) )
@@ -422,21 +562,48 @@
         main.log.report( "Add 300 host intents and verify pingall" )
         main.log.report( "_______________________________________" )
         import itertools
-
+        
         main.case( "Install 300 host intents" )
         main.step( "Add host Intents" )
         intentResult = main.TRUE
-        hostCombos = list( itertools.combinations( main.hostMACs, 2 ) )
-        for i in range( len( hostCombos ) ):
-            iResult = main.ONOScli1.addHostIntent(
-                hostCombos[ i ][ 0 ],
-                hostCombos[ i ][ 1 ] )
-            intentResult = ( intentResult and iResult )
+        hostCombos = list( itertools.combinations( main.hostMACs, 2 ) ) 
+        
+        CLI1 = (main.ONOScli1.addHostIntent)
+        CLI2 = (main.ONOScli2.addHostIntent)
+        CLI3 = (main.ONOScli3.addHostIntent)
+        CLI4 = (main.ONOScli4.addHostIntent)
+        CLI5 = (main.ONOScli5.addHostIntent)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        results = main.TRUE
+        intentIdList = []
+        time1 = time.time()
+        for i in xrange(0,len(hostCombos),5):
+            pool = []
+            for cli in ONOSCLI:
+                if i >= len(hostCombos):
+                    break
+                t = main.Thread(target=cli,threadID=main.threadID,
+                        name="addHostIntent",
+                        args=[hostCombos[i][0],hostCombos[i][1]])
+                pool.append(t)
+                t.start()
+                i = i + 1
+                main.threadID = main.threadID + 1
+            for thread in pool:
+                thread.join()
+                intentIdList.append(thread.result)
+        time2 = time.time()
+        main.log.info("Time for adding host intents: %2f seconds" %(time2-time1))
+        intentResult = main.TRUE
+        intentsJson = main.ONOScli2.intents()
+        getIntentResult = main.ONOScli1.getIntentState(intentsId = intentIdList,
+                intentsJson = intentsJson)
+        print getIntentResult
 
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall()
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -448,6 +615,7 @@
                                  onfail="PING ALL FAIL" )
 
         case4Result = ( intentResult and pingResult )
+        
         #case4Result = pingResult
         utilities.assert_equals(
             expect=main.TRUE,
@@ -518,7 +686,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkDown = main.FALSE
         time1 = time.time()
-        pingResultLinkDown = main.Mininet1.pingall()
+        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -555,15 +723,15 @@
         for i in range( int( switchLinksToToggle ) ):
             main.Mininet1.link(
                 END1=link1End1,
-                END2=randomLink1[ i ],
+                END2=main.randomLink1[ i ],
                 OPTION="up" )
             main.Mininet1.link(
                 END1=link2End1,
-                END2=randomLink2[ i ],
+                END2=main.randomLink2[ i ],
                 OPTION="up" )
             main.Mininet1.link(
                 END1=link3End1,
-                END2=randomLink3[ i ],
+                END2=main.randomLink3[ i ],
                 OPTION="up" )
         time.sleep( link_sleep )
 
@@ -840,10 +1008,43 @@
                 intentsTemp = intentsList[ i ].split( ',' )
                 intentIdList.append( intentsTemp[ 0 ] )
             print "Intent IDs: ", intentIdList
-            for id in range( len( intentIdList ) ):
+            
+            
+            CLI1 = (main.ONOScli1.removeIntent)
+            CLI2 = (main.ONOScli2.removeIntent)
+            CLI3 = (main.ONOScli3.removeIntent)
+            CLI4 = (main.ONOScli4.removeIntent)
+            CLI5 = (main.ONOScli5.removeIntent)
+            ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+            results = main.TRUE
+            time1 = time.time()
+            
+            for i in xrange(0,len(intentIdList),5):
+                pool = []
+                for cli in ONOSCLI:
+                    if i >= len(intentIdList):
+                        break
+                    print "Removing intent id (round 1) :", intentIdList[ i ]
+                    t = main.Thread(target=cli,threadID=main.threadID,
+                            name="removeIntent",
+                            args=[intentIdList[i]])
+                    pool.append(t)
+                    t.start()
+                    i = i + 1
+                    main.threadID = main.threadID + 1
+                    
+                for thread in pool:
+                    thread.join()
+                    results = results and thread.result
+ 
+            time2 = time.time()
+            main.log.info("Time for feature:install onos-app-fwd: %2f seconds" %(time2-time1))
+            """
+                for id in range( len( intentIdList ) ):
                 print "Removing intent id (round 1) :", intentIdList[ id ]
                 main.ONOScli1.removeIntent( intentId=intentIdList[ id ] )
                 #time.sleep( 1 )
+            """
 
             main.log.info(
                 "Verify all intents are removed and if any leftovers try remove one more time" )
@@ -863,14 +1064,28 @@
             intentIdList1 = []
             if ( len( intentsList1 ) > 1 ):
                 for i in range( len( intentsList1 ) ):
-                    intentsTemp1 = intentsList[ i ].split( ',' )
+                    intentsTemp1 = intentsList1[ i ].split( ',' )
                     intentIdList1.append( intentsTemp1[ 0 ] )
                 print "Leftover Intent IDs: ", intentIdList1
-                for id in range( len( intentIdList1 ) ):
-                    print "Removing intent id (round 2):", intentIdList1[ id ]
-                    main.ONOScli1.removeIntent(
-                        intentId=intentIdList1[ id ] )
-                    #time.sleep( 2 )
+ 
+                for i in xrange(0,len(intentIdList1),5):
+                    pool = []
+                    for cli in ONOSCLI:
+                        if i >= len(intentIdList):
+                            break
+                        print "Removing intent id (round 2) :", intentIdList1[ i ]
+                        t = main.Thread(target=cli,threadID=main.threadID,
+                                name="removeIntent",
+                                args=[intentIdList1[i]])
+                        pool.append(t)
+                        t.start()
+                        i = i + 1
+                        main.threadID = main.threadID + 1
+                        
+                    for thread in pool:
+                        thread.join()
+                        results = results and thread.result
+                step1Result = results
             else:
                 print "There are no more intents that need to be removed"
                 step1Result = main.TRUE
@@ -891,47 +1106,93 @@
         import copy
         import time
 
+        Thread = imp.load_source('Thread','/home/admin/ONLabTest/TestON/tests/OnosCHO/Thread.py')
+        threadID = 0
+
         main.log.report( "Enable Intent based Reactive forwarding and Verify ping all" )
         main.log.report( "_____________________________________________________" )
         main.case( "Enable Intent based Reactive forwarding and Verify ping all" )
         main.step( "Enable intent based Reactive forwarding" )
-        installResult = main.TRUE
-        for i in range( 1, int( main.numCtrls ) + 1 ):
-            onosFeature = 'onos-app-ifwd'
-            ONOS_ip = main.params[ 'CTRL' ][ 'ip' + str( i ) ]
-            ONOScli = 'ONOScli' + str( i )
-            main.log.info( "Enabling Intent based Reactive forwarding on ONOS Node " + ONOS_ip )
-            exec "inResult=main." + ONOScli + ".featureInstall(onosFeature)"
-            time.sleep( 3 )
-            installResult = inResult and installResult
-
-        time.sleep( 5 )
-
+        installResult = main.FALSE
+        feature = "onos-app-ifwd"
+        CLI1 = (main.ONOScli1.featureInstall,feature)
+        CLI2 = (main.ONOScli2.featureInstall,feature)
+        CLI3 = (main.ONOScli3.featureInstall,feature)
+        CLI4 = (main.ONOScli4.featureInstall,feature)
+        CLI5 = (main.ONOScli5.featureInstall,feature)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        pool = []
+        time1 = time.time()
+        for cli,feature in ONOSCLI:
+            t = main.Thread(target=cli,threadID=threadID,
+                    name="featureInstall",args=[feature])
+            pool.append(t)
+            t.start()
+            threadID = threadID + 1
+            
+        results = []
+        for thread in pool:
+            thread.join()
+            results.append(thread.result)
+        time2 = time.time()
+        
+        if( all(result == main.TRUE for result in results) == False):
+                main.log.info("Did not install onos-app-ifwd feature properly")
+                main.cleanup()
+                main.exit()
+        else:
+            main.log.info("Successful feature:install onos-app-ifwd")
+            installResult = main.TRUE
+        main.log.info("Time for feature:install onos-app-ifwd: %2f seconds" %(time2-time1))
+        
         main.step( "Verify Pingall" )
         ping_result = main.FALSE
         time1 = time.time()
-        ping_result = main.Mininet1.pingall()
+        ping_result = main.Mininet1.pingall(timeout=600)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
             "Time taken for Ping All: " +
             str( timeDiff ) +
             " seconds" )
-
+        
         if ping_result == main.TRUE:
             main.log.report( "Pingall Test in Reactive mode successful" )
         else:
             main.log.report( "Pingall Test in Reactive mode failed" )
 
         main.step( "Disable Intent based Reactive forwarding" )
-        uninstallResult = main.TRUE
-        for i in range( 1, int( main.numCtrls ) + 1 ):
-            onosFeature = 'onos-app-ifwd'
-            ONOS_ip = main.params[ 'CTRL' ][ 'ip' + str( i ) ]
-            ONOScli = 'ONOScli' + str( i )
-            main.log.info( "Disabling Intent based Reactive forwarding on ONOS Node " + ONOS_ip )
-            exec "unResult=main." + ONOScli + ".featureUninstall(onosFeature)"
-            uninstallResult = unResult and uninstallResult
+        uninstallResult = main.FALSE
+        
+        CLI1 = (main.ONOScli1.featureUninstall,feature)
+        CLI2 = (main.ONOScli2.featureUninstall,feature)
+        CLI3 = (main.ONOScli3.featureUninstall,feature)
+        CLI4 = (main.ONOScli4.featureUninstall,feature)
+        CLI5 = (main.ONOScli5.featureUninstall,feature)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        pool = []
+        time1 = time.time()
+        for cli,feature in ONOSCLI:
+            t = main.Thread(target=cli,threadID=threadID,
+                    name="featureUninstall",args=[feature])
+            pool.append(t)
+            t.start()
+            threadID = threadID + 1
+            
+        results = []
+        for thread in pool:
+            thread.join()
+            results.append(thread.result)
+        time2 = time.time()
+        
+        if( all(result == main.TRUE for result in results) == False):
+                main.log.info("Did not uninstall onos-app-ifwd feature properly")
+                main.cleanup()
+                main.exit()
+        else:
+            main.log.info("Successful feature:uninstall onos-app-ifwd")
+            uninstallResult = main.TRUE
+        main.log.info("Time for feature:uninstall onos-app-ifwd: %2f seconds" %(time2-time1))
 
         # Waiting for reative flows to be cleared.
         time.sleep( 10 )
@@ -948,12 +1209,14 @@
         import re
         import time
         import copy
+        
 
+        Thread = imp.load_source('Thread','/home/admin/ONLabTest/TestON/tests/OnosCHO/Thread.py')
         newTopo = main.params['TOPO2']['topo']
         main.numMNswitches = int ( main.params[ 'TOPO2' ][ 'numSwitches' ] )
         main.numMNlinks = int ( main.params[ 'TOPO2' ][ 'numLinks' ] )
         main.numMNhosts = int ( main.params[ 'TOPO2' ][ 'numHosts' ] )
-
+        main.pingTimeout = 60
         main.log.report(
             "Load Chordal topology and Balance all Mininet switches across controllers" )
         main.log.report(
@@ -1027,16 +1290,39 @@
         #karafTimeout = "3600000" # This is not needed here as it is already set before.
         # need to wait here sometime for ONOS to bootup.
         time.sleep( 30 )
-        for i in range( 1, int( main.numCtrls ) + 1 ):
-            ONOS_ip = main.params[ 'CTRL' ][ 'ip' + str( i ) ]
-            ONOScli = 'ONOScli' + str( i )
-            main.log.info( "ONOS Node " + ONOS_ip + " cli start:" )
-            exec "startcli=main." + ONOScli + \
-                ".startOnosCli(ONOS_ip)"
-            utilities.assert_equals( expect=main.TRUE, actual=startcli,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            cliResult = ( cliResult and startcli )
+
+        main.log.step(" Start ONOS cli using thread ")
+        
+        CLI1 = (main.ONOScli1.startOnosCli,main.ONOS1_ip)
+        CLI2 = (main.ONOScli2.startOnosCli,main.ONOS2_ip)
+        CLI3 = (main.ONOScli3.startOnosCli,main.ONOS3_ip)
+        CLI4 = (main.ONOScli4.startOnosCli,main.ONOS4_ip)
+        CLI5 = (main.ONOScli5.startOnosCli,main.ONOS5_ip)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        pool = []
+        time1 = time.time()
+        for cli,ip in ONOSCLI:
+            t = main.Thread(target=cli,threadID=main.threadID,
+                    name="startOnosCli",args=[ip])
+            pool.append(t)
+            t.start()
+            main.threadID = main.threadID + 1
+            
+        cliResult  = main.FALSE
+        results = []
+        for thread in pool:
+            thread.join()
+            results.append(thread.result)
+        time2 = time.time()
+        
+        if( all(result == main.TRUE for result in results) == False):
+                main.log.info("ONOS CLI did not start up properly")
+                main.cleanup()
+                main.exit()
+        else:
+            main.log.info("Successful CLI startup")
+            cliResult = main.TRUE
+        main.log.info("Time for connecting to CLI: %2f seconds" %(time2-time1))
 
         main.step( "Balance devices across controllers" )
         for i in range( int( main.numCtrls ) ):
@@ -1063,7 +1349,7 @@
         main.numMNswitches = int ( main.params[ 'TOPO3' ][ 'numSwitches' ] )
         main.numMNlinks = int ( main.params[ 'TOPO3' ][ 'numLinks' ] )
         main.numMNhosts = int ( main.params[ 'TOPO3' ][ 'numHosts' ] )
-
+        main.pingTimeout = 600
         main.log.report(
             "Load Spine and Leaf topology and Balance all Mininet switches across controllers" )
         main.log.report(
@@ -1137,16 +1423,45 @@
         #karafTimeout = "3600000" # This is not needed here as it is already set before.
         # need to wait here sometime for ONOS to bootup.
         time.sleep( 30 )
-        for i in range( 1, int( main.numCtrls ) + 1 ):
-            ONOS_ip = main.params[ 'CTRL' ][ 'ip' + str( i ) ]
-            ONOScli = 'ONOScli' + str( i )
-            main.log.info( "ONOS Node " + ONOS_ip + " cli start:" )
-            exec "startcli=main." + ONOScli + \
-                ".startOnosCli(ONOS_ip)"
-            utilities.assert_equals( expect=main.TRUE, actual=startcli,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            cliResult = ( cliResult and startcli )
+        
+        main.log.step(" Start ONOS cli using thread ")
+        
+        CLI1 = (main.ONOScli1.startOnosCli,main.ONOS1_ip)
+        CLI2 = (main.ONOScli2.startOnosCli,main.ONOS2_ip)
+        CLI3 = (main.ONOScli3.startOnosCli,main.ONOS3_ip)
+        CLI4 = (main.ONOScli4.startOnosCli,main.ONOS4_ip)
+        CLI5 = (main.ONOScli5.startOnosCli,main.ONOS5_ip)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        pool = []
+        time1 = time.time()
+        for cli,ip in ONOSCLI:
+            t = main.Thread(target=cli,threadID=main.threadID,
+                    name="startOnosCli",args=[ip])
+            pool.append(t)
+            t.start()
+            main.threadID = main.threadID + 1
+            
+        cliResult  = main.FALSE
+        results = []
+        for thread in pool:
+            thread.join()
+            results.append(thread.result)
+        time2 = time.time()
+        
+        if( all(result == main.TRUE for result in results) == False):
+                main.log.info("ONOS CLI did not start up properly")
+                main.cleanup()
+                main.exit()
+        else:
+            main.log.info("Successful CLI startup")
+            cliResult = main.TRUE
+        main.log.info("Time for connecting to CLI: %2f seconds" %(time2-time1))
+
+        main.step( "Balance devices across controllers" )
+        for i in range( int( main.numCtrls ) ):
+            balanceResult = main.ONOScli1.balanceMasters()
+            # giving some breathing time for ONOS to complete re-balance
+            time.sleep( 3 )
 
         main.step( "Balance devices across controllers" )
         for i in range( int( main.numCtrls ) ):
@@ -1160,3 +1475,149 @@
             actual=case13Result,
             onpass="Starting new Spine topology test PASS",
             onfail="Starting new Spine topology test FAIL" )
+
+    def CASE14(self,main):
+        """
+        Install 300 host intents and verify ping all
+        """
+        main.log.report( "Add 300 host intents and verify pingall" )
+        main.log.report( "_______________________________________" )
+        import itertools
+        
+        main.case( "Install 300 host intents" )
+        main.step( "Add host Intents" )
+        intentResult = main.TRUE
+        hostCombos = list( itertools.combinations( main.hostMACs, 2 ) ) 
+        
+        CLI1 = (main.ONOScli1.addHostIntent)
+        CLI2 = (main.ONOScli2.addHostIntent)
+        CLI3 = (main.ONOScli3.addHostIntent)
+        CLI4 = (main.ONOScli4.addHostIntent)
+        CLI5 = (main.ONOScli5.addHostIntent)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        results = main.TRUE
+        time1 = time.time()
+        for i in xrange(0,len(hostCombos),5):
+            pool = []
+            for cli in ONOSCLI:
+                if i >= len(hostCombos):
+                    break
+                t = main.Thread(target=cli,threadID=main.threadID,
+                        name="addHostIntent",
+                        args=[hostCombos[i][0],hostCombos[i][1]])
+                pool.append(t)
+                t.start()
+                i = i + 1
+                main.threadID = main.threadID + 1
+                
+            for thread in pool:
+                thread.join()
+                results = results and thread.result
+
+        time2 = time.time()
+        
+        main.log.info("Time for adding host intents: %2f seconds" %(time2-time1))
+        intentResult = results
+        """
+        for i in range( len( hostCombos ) ):
+            iResult = main.ONOScli1.addHostIntent(
+                hostCombos[ i ][ 0 ],
+                hostCombos[ i ][ 1 ] )
+            intentResult = ( intentResult and iResult )
+        """
+        
+        main.step( "Verify Ping across all hosts" )
+        pingResult = main.FALSE
+        time1 = time.time()
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
+        time2 = time.time()
+        timeDiff = round( ( time2 - time1 ), 2 )
+        main.log.report(
+            "Time taken for Ping All: " +
+            str( timeDiff ) +
+            " seconds" )
+        utilities.assert_equals( expect=main.TRUE, actual=pingResult,
+                                 onpass="PING ALL PASS",
+                                 onfail="PING ALL FAIL" )
+
+        case4Result = ( intentResult and pingResult )
+        
+        #case4Result = pingResult
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=case4Result,
+            onpass="Install 300 Host Intents and Ping All test PASS",
+            onfail="Install 300 Host Intents and Ping All test FAIL" )
+    
+    def CASE15( self,main):
+        """
+        Install 300 host intents and verify ping all
+        """
+        main.log.report( "Add 300 host intents and verify pingall" )
+        main.log.report( "_______________________________________" )
+        import itertools
+
+        main.case( "Install 300 host intents" )
+        main.step( "Add host Intents" )
+        intentResult = main.TRUE
+        hostCombos = list( itertools.combinations( main.hostMACs, 2 ) ) 
+        
+        CLI1 = (main.ONOScli1.addHostIntent)
+        CLI2 = (main.ONOScli2.addHostIntent)
+        CLI3 = (main.ONOScli3.addHostIntent)
+        CLI4 = (main.ONOScli4.addHostIntent)
+        CLI5 = (main.ONOScli5.addHostIntent)
+        ONOSCLI = [CLI1,CLI2,CLI3,CLI4,CLI5]
+        results = main.TRUE
+        time1 = time.time()
+        for i in xrange(0,len(hostCombos),5):
+            pool = []
+            for cli in ONOSCLI:
+                if i >= len(hostCombos):
+                    break
+                t = main.Thread(target=cli,threadID=main.threadID,
+                        name="addHostIntent",
+                        args=[hostCombos[i][0],hostCombos[i][1]])
+                pool.append(t)
+                t.start()
+                i = i + 1
+                main.threadID = main.threadID + 1
+                
+            for thread in pool:
+                thread.join()
+                results = results and thread.result
+
+        time2 = time.time()
+        
+        main.log.info("Time for adding host intents: %2f seconds" %(time2-time1))
+        intentResult = results
+        """
+        for i in range( len( hostCombos ) ):
+            iResult = main.ONOScli1.addHostIntent(
+                hostCombos[ i ][ 0 ],
+                hostCombos[ i ][ 1 ] )
+            intentResult = ( intentResult and iResult )
+        """
+        
+        main.step( "Verify Ping across all hosts" )
+        pingResult = main.FALSE
+        time1 = time.time()
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
+        time2 = time.time()
+        timeDiff = round( ( time2 - time1 ), 2 )
+        main.log.report(
+            "Time taken for Ping All: " +
+            str( timeDiff ) +
+            " seconds" )
+        utilities.assert_equals( expect=main.TRUE, actual=pingResult,
+                                 onpass="PING ALL PASS",
+                                 onfail="PING ALL FAIL" )
+
+        case4Result = ( intentResult and pingResult )
+        
+        #case4Result = pingResult
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=case4Result,
+            onpass="Install 300 Host Intents and Ping All test PASS",
+            onfail="Install 300 Host Intents and Ping All test FAIL" )