Merge branch 'devl/newOnosCHO' of https://github.com/opennetworkinglab/ONLabTest into devl/newOnosCHO
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 575e6c2..d59adbe 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1748,27 +1748,27 @@
                 intentsJsonTemp = json.loads( intentsJson )
             if isinstance( intentsId, types.StringType ):
                 for intent in intentsJsonTemp:
-                    if intentsId == intent['id']:
-                        state = intent['state']
+                    if intentsId == intent[ 'id' ]:
+                        state = intent[ 'state' ]
                         return state
                 main.log.info( "Cannot find intent ID" + str( intentsId ) +
                                " on the list" )
                 return state
             elif isinstance( intentsId, types.ListType ):
                 dictList = []
-                for ID in intentsId:
+                for i in xrange( len( intentsId ) ):
                     stateDict = {}
                     for intents in intentsJsonTemp:
-                        if ID == intents['id']:
-                            stateDict['state'] = intents['state']
-                            stateDict['id'] = ID
+                        if intentsId[ i ] == intents[ 'id' ]:
+                            stateDict[ 'state' ] = intents[ 'state' ]
+                            stateDict[ 'id' ] = intentsId[ i ]
                             dictList.append( stateDict )
                             break
                 if len( intentsId ) != len( dictList ):
                     main.log.info( "Cannot find some of the intent ID state" )
                 return dictList
             else:
-                main.log.info("Invalid intents ID entry")
+                main.log.info( "Invalid intents ID entry" )
                 return None
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
@@ -1782,7 +1782,56 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanup()
             main.exit()
+    
+    def checkIntentState( self, intentsId, expectedState = 'INSTALLED' ):
+        """
+        Description:
+            Check intents state
+        Required:
+            intentsId - List of intents ID to be checked
+        Optional:
+            expectedState - Check this expected state of each intents state
+                            in the list. Defaults to INSTALLED
+        Return:
+            Returns main.TRUE only if all intent are the same as expectedState,
+            , otherwise,returns main.FALSE.
+        """
+        try:
+            # Generating a dictionary: intent id as a key and state as value
+            intentsDict = self.getIntentState( intentsId )
+            print "len of intentsDict ", str( len( intentsDict ) )
+            if len( intentsId ) != len( intentsDict ):
+                main.log.info( self.name + "There is something wrong " +
+                               "getting intents state" )
+                return main.FALSE
+            returnValue = main.TRUE
+            for intents in intentsDict:
+                if intents.get( 'state' ) != expectedState:
+                    main.log.info( self.name + " : " + intents.get( 'id' ) +
+                                   " actual state = " + intents.get( 'state' )
+                                   + " does not equal expected state = "
+                                   + expectedState )
+                    returnValue = main.FALSE
+            if returnValue == main.TRUE:
+                main.log.info( self.name + ": All " +
+                               str( len( intentsDict ) ) +
+                               " intents are in " + expectedState + " state")
+            return returnValue
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+        return main.TRUE
 
+    
     def flows( self, jsonFormat=True ):
         """
         Optional:
diff --git a/TestON/tests/OnosCHO/OnosCHO.py b/TestON/tests/OnosCHO/OnosCHO.py
index cd70c80..547cb0a 100644
--- a/TestON/tests/OnosCHO/OnosCHO.py
+++ b/TestON/tests/OnosCHO/OnosCHO.py
@@ -195,7 +195,7 @@
         for i in range( 1, ( main.numMNswitches + 1 ) ):  # 1 to ( num of switches +1 )
             main.Mininet1.assignSwController(
                 sw=str( i ),
-                count=int( main.numCtrls ),
+                count= 1 ,
                 ip1=main.ONOS1_ip,
                 port1=main.ONOS1_port,
                 ip2=main.ONOS2_ip,
@@ -206,7 +206,7 @@
                 port4=main.ONOS4_port,
                 ip5=main.ONOS5_ip,
                 port5=main.ONOS5_port )
-
+            time.sleep(2)
         switch_mastership = main.TRUE
         for i in range( 1, ( main.numMNswitches + 1 ) ):
             response = main.Mininet1.getSwController( "s" + str( i ) )
@@ -242,13 +242,14 @@
         """
         time.sleep(15)
         #Don't balance master for now..
-        main.step( "Balance devices across controllers" )
+        """main.step( "Balance devices across controllers" )
         for i in range( int( main.numCtrls ) ):
             balanceResult = main.ONOScli1.balanceMasters()
             # giving some breathing time for ONOS to complete re-balance
             time.sleep( 3 )
         topology_output = main.ONOScli1.topology()
         topology_result = main.ONOSbench.getTopology( topology_output )
+        """
         case2Result = ( switch_mastership and startStatus )
         utilities.assert_equals(
             expect=main.TRUE,
@@ -350,7 +351,7 @@
         main.case(
             "Assign and Balance all Mininet switches across controllers" )
         main.step( "Stop any previous Mininet network topology" )
-        stopStatus = main.Mininet1.stopNet(fileName = "topoSpine" )
+        #stopStatus = main.Mininet1.stopNet(fileName = "topoSpine" )
         main.step( "Start Mininet with Spine topology" )
         startStatus = main.Mininet1.startNet(topoFile = main.newTopo)
         time.sleep(20)
@@ -564,7 +565,7 @@
         main.step( "Verify Pingall" )
         ping_result = main.FALSE
         time1 = time.time()
-        ping_result = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        ping_result = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -644,7 +645,7 @@
         main.step( "Verify Pingall" )
         ping_result = main.FALSE
         time1 = time.time()
-        ping_result = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        ping_result = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -724,7 +725,7 @@
         main.step( "Verify Pingall" )
         ping_result = main.FALSE
         time1 = time.time()
-        ping_result = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        ping_result = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -901,13 +902,21 @@
         intentsJson = main.ONOScli2.intents()
         getIntentStateResult = main.ONOScli1.getIntentState(intentsId = intentIdList,
                 intentsJson = intentsJson)
+        print "len of intent ID", str(len(intentIdList))
+        print "len of intent state results", str(len(getIntentStateResult))
         print getIntentStateResult
         # Takes awhile for all the onos to get the intents
-        time.sleep(30)
+        time.sleep( 15 )
+        """intentState = main.TRUE
+        for i in getIntentStateResult:
+            if getIntentStateResult.get( 'state' ) != 'INSTALLED':
+        """
+
+
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -969,7 +978,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1030,7 +1039,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1112,7 +1121,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkDown = main.FALSE
         time1 = time.time()
-        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout )
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1177,7 +1186,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkUp = main.FALSE
         time1 = time.time()
-        pingResultLinkUp = main.Mininet1.pingall( timeout=main.pingTimeout,shortCircuit=True )
+        pingResultLinkUp = main.Mininet1.pingall( timeout=main.pingTimeout )
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1256,7 +1265,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkDown = main.FALSE
         time1 = time.time()
-        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1321,7 +1330,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkUp = main.FALSE
         time1 = time.time()
-        pingResultLinkUp = main.Mininet1.pingall(timeout = main.pingTimeout, shortCircuit = True )
+        pingResultLinkUp = main.Mininet1.pingall(timeout = main.pingTimeout )
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1379,7 +1388,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkDown = main.FALSE
         time1 = time.time()
-        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1434,7 +1443,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkUp = main.FALSE
         time1 = time.time()
-        pingResultLinkUp = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResultLinkUp = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1492,7 +1501,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkDown = main.FALSE
         time1 = time.time()
-        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1547,7 +1556,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkUp = main.FALSE
         time1 = time.time()
-        pingResultLinkUp = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResultLinkUp = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1614,7 +1623,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkDown = main.FALSE
         time1 = time.time()
-        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResultLinkDown = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1667,7 +1676,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResultLinkUp = main.FALSE
         time1 = time.time()
-        pingResultLinkUp = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResultLinkUp = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1727,7 +1736,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1790,7 +1799,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1820,12 +1829,11 @@
         main.case( "Install 4556 point intents" )
         main.step( "Add point Intents" )
         intentResult = main.TRUE
-        main.pingTimeout = 600 
+        main.pingTimeout = 600
         for i in range(len(main.hostMACs)):
             main.MACsDict[main.deviceDPIDs[i+10]] = main.hostMACs[i].split('/')[0]
         print main.MACsDict
-        deviceCombos = list( itertools.permutations( main.deviceDPIDs[10:], 2 ) ) 
-        
+        deviceCombos = list( itertools.permutations( main.deviceDPIDs[10:], 2 ) )
         intentIdList = []
         time1 = time.time()
         for i in xrange( 0, len( deviceCombos ), int(main.numCtrls) ):
@@ -1857,7 +1865,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1912,13 +1920,26 @@
                 thread.join()
                 intentIdList.append(thread.result)
         time2 = time.time()
-        main.log.info("Time for adding point intents: %2f seconds" %(time2-time1)) 
+        main.log.info("Time for adding point intents: %2f seconds" %(time2-time1))
+        time.sleep(30)
+        print "getting all intents ID"
+        intentIdTemp = main.ONOScli1.getAllIntentsId()
+        print intentIdTemp
+        print len(intentIdList)
         print intentIdList
-        time.sleep(5)
+        print "Checking intents state"
+        checkIntentStateResult = main.ONOScli1.checkIntentState( intentsId = intentIdList )
+        #checkIntentStateResult = main.ONOScli2.checkIntentState( intentsId = intentIdList )
+        #checkIntentStateResult = main.ONOScli3.checkIntentState( intentsId = intentIdList )
+        #checkIntentStateResult = main.ONOScli4.checkIntentState( intentsId = intentIdList )
+        #checkIntentStateResult = main.ONOScli5.checkIntentState( intentsId = intentIdList )
+        if checkIntentStateResult:
+            main.log.info( "All intents are installed correctly " )
+        time.sleep(50)
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -1974,7 +1995,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -2031,7 +2052,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -2086,7 +2107,7 @@
         main.step( "Verify Ping across all hosts" )
         pingResult = main.FALSE
         time1 = time.time()
-        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True)
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
         time2 = time.time()
         timeDiff = round( ( time2 - time1 ), 2 )
         main.log.report(
@@ -2101,6 +2122,68 @@
             onpass="Install 25 single to multi point Intents and Ping All test PASS",
             onfail="Install 25 single to multi point Intents and Ping All test FAIL" )
 
+    def CASE98( self ):
+        """
+        Install single-multi point intents and verify Ping all works
+        for Spine topology
+        """
+        import copy
+        main.log.report( "Install single-multi point intents and verify Ping all" )
+        main.log.report( "___________________________________________" )
+        main.case( "Install single-multi point intents and Ping all" )
+        deviceDPIDsCopy = copy.copy( main.deviceDPIDs )
+        deviceDPIDsCopy = deviceDPIDsCopy[ 10: ]
+        portEgressList = [ '1' ]*(len(deviceDPIDsCopy) - 1)
+        intentIdList = []
+        MACsDictCopy = {}
+        for i in range( len( deviceDPIDsCopy ) ):
+            MACsDictCopy[ deviceDPIDsCopy[ i ] ] = main.hostMACs[i].split( '/' )[ 0 ]
+
+        print "deviceDPIDsCopy", deviceDPIDsCopy
+        print ""
+        print "MACsDictCopy", MACsDictCopy
+        time1 = time.time()
+        for i in xrange(0,len(deviceDPIDsCopy),int(main.numCtrls)):
+            pool = []
+            for cli in main.CLIs:
+                if i >= len( deviceDPIDsCopy ):
+                    break
+                ingressDevice = deviceDPIDsCopy[i]
+                egressDeviceList = copy.copy(deviceDPIDsCopy)
+                egressDeviceList.remove(ingressDevice)
+                t = main.Thread( target=cli.addSinglepointToMultipointIntent,
+                        threadID=main.threadID,
+                        name="addSinglepointToMultipointIntent",
+                        args =[ingressDevice,egressDeviceList,'1',portEgressList,'IPV4',MACsDictCopy.get(ingressDevice),''])
+                pool.append(t)
+                #time.sleep(1)
+                t.start()
+                i = i + 1
+                main.threadID = main.threadID + 1
+            for thread in pool:
+                thread.join()
+                intentIdList.append(thread.result)
+        time2 = time.time()
+        main.log.info("Time for adding point intents: %2f seconds" %(time2-time1)) 
+        time.sleep(5)
+        main.step( "Verify Ping across all hosts" )
+        pingResult = main.FALSE
+        time1 = time.time()
+        pingResult = main.Mininet1.pingall(timeout=main.pingTimeout,shortCircuit=True,acceptableFailed=5)
+        time2 = time.time()
+        timeDiff = round( ( time2 - time1 ), 2 )
+        main.log.report(
+            "Time taken for Ping All: " +
+            str( timeDiff ) +
+            " seconds" )
+
+        case98Result = pingResult
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=case98Result,
+            onpass="Install 25 single to multi point Intents and Ping All test PASS",
+            onfail="Install 25 single to multi point Intents and Ping All test FAIL" )
+
     def CASE10( self ):
         import time
         """
@@ -2199,7 +2282,7 @@
                                  onpass="Intent removal test successful",
                                  onfail="Intent removal test failed" )
 
-    def CASE11( self, main ):
+    def CASE12( self, main ):
         """
         Enable onos-app-ifwd, Verify Intent based Reactive forwarding through ping all and Disable it
         """
@@ -2356,3 +2439,4 @@
 
 
 
+
diff --git a/TestON/tests/ProdFunc/ProdFunc.py b/TestON/tests/ProdFunc/ProdFunc.py
index c64ec78..f779b6b 100644
--- a/TestON/tests/ProdFunc/ProdFunc.py
+++ b/TestON/tests/ProdFunc/ProdFunc.py
@@ -812,9 +812,6 @@
             actual=case12Result,
             onpass = "Expected default num of flows exist",
             onfail = "Expected default num of flows do not exist")
-    
-    
-        
 
     def CASE6( self ):
         import time
@@ -940,6 +937,9 @@
             onfail="Pingall Test after Host intents addition failed" )
 
     def CASE5( self, main ):
+        """
+            Check ONOS topology matches with mininet
+        """
         import json
         # assumes that sts is already in you PYTHONPATH
         from sts.topology.teston_topology import TestONTopology
@@ -1052,6 +1052,11 @@
             onfail="Topology checks failed" )
 
     def CASE7( self, main ):
+        """
+            Link discovery test case. Checks if ONOS can discover a link
+            down or up properly.
+        """
+
         from sts.topology.teston_topology import TestONTopology
 
         linkSleep = int( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1280,8 +1285,11 @@
                                  onfail="Intent removal test failed" )
 
     def CASE9( self ):
+        """
+            Testing Point intents
+        """
         main.log.report(
-            "This testcase adds point intents and then does pingall" )
+            "This test case adds point intents and then does pingall" )
         main.log.report( "__________________________________" )
         main.log.info( "Adding point intents" )
         main.case(