Merge "[ONOS-6153] Refactor intent reroute test"
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 673b7ce..18f0914 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -163,7 +163,7 @@
                         except pexpect.TIMEOUT:
                             main.log.error( "ONOS did not respond to 'logout' or CTRL-d" )
                         return main.TRUE
-                    else: # some other output
+                    else:  # some other output
                         main.log.warn( "Unknown repsonse to logout command: '{}'",
                                        repr( self.handle.before ) )
                         return main.FALSE
@@ -488,7 +488,7 @@
             if debug:
                 # NOTE: This adds and average of .4 seconds per call
                 logStr = "\"Sending CLI command: '" + cmdStr + "'\""
-                self.log( logStr,noExit=noExit )
+                self.log( logStr, noExit=noExit )
             self.handle.sendline( cmdStr )
             i = self.handle.expect( ["onos>", "\$"], timeout )
             response = self.handle.before
@@ -853,8 +853,8 @@
                     main.log.error( "Mastership not balanced" )
                     main.log.info( "\n" + self.checkMasters( False ) )
                     return main.FALSE
-            main.log.info( "Mastership balanced between " \
-                            + str( len(masters) ) + " masters" )
+            main.log.info( "Mastership balanced between " +
+                           str( len(masters) ) + " masters" )
             return main.TRUE
         except ( TypeError, ValueError ):
             main.log.exception( "{}: Object not as expected: {!r}".format( self.name, mastersOutput ) )
@@ -2254,7 +2254,7 @@
                                str( len( intentDictONOS ) ) + " actual" )
                 returnValue = main.FALSE
             for intentID in intentDict.keys():
-                if not intentID in intentDictONOS.keys():
+                if intentID not in intentDictONOS.keys():
                     main.log.debug( self.name + ": intent ID - " + intentID + " is not in ONOS" )
                     returnValue = main.FALSE
                 else:
@@ -2304,7 +2304,7 @@
 
             # Check response if something wrong
             response = self.sendline( cmd, timeout=timeout, noExit=noExit )
-            if response == None:
+            if response is None:
                 return main.FALSE
             response = json.loads( response )
 
@@ -2327,6 +2327,9 @@
             else:
                 main.cleanup()
                 main.exit()
+        except pexpect.TIMEOUT:
+            main.log.error( self.name + ": ONOS timeout" )
+            return None
         except Exception:
             main.log.exception( self.name + ": Uncaught exception!" )
             if noExit:
@@ -2334,9 +2337,6 @@
             else:
                 main.cleanup()
                 main.exit()
-        except pexpect.TIMEOUT:
-            main.log.error( self.name + ": ONOS timeout" )
-            return None
 
     def flows( self, state="", jsonFormat=True, timeout=60, noExit=False, noCore=False ):
         """
@@ -2381,10 +2381,10 @@
 
     def checkFlowCount(self, min=0, timeout=60 ):
         count = self.getTotalFlowsNum( timeout=timeout )
-        count = int (count) if count else 0
-        return count if (count > min) else False
+        count = int( count ) if count else 0
+        return count if ( count > min ) else False
 
-    def checkFlowsState( self, isPENDING=True, timeout=60,noExit=False ):
+    def checkFlowsState( self, isPENDING=True, timeout=60, noExit=False ):
         """
         Description:
             Check the if all the current flows are in ADDED state
@@ -2433,6 +2433,9 @@
         except AssertionError:
             main.log.exception( "" )
             return None
+        except pexpect.TIMEOUT:
+            main.log.error( self.name + ": ONOS timeout" )
+            return None
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
             main.log.error( self.name + ":    " + self.handle.before )
@@ -2442,10 +2445,6 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanup()
             main.exit()
-        except pexpect.TIMEOUT:
-            main.log.error( self.name + ": ONOS timeout" )
-            return None
-
 
     def pushTestIntents( self, ingress, egress, batchSize, offset="",
                          options="", timeout=10, background = False, noExit=False, getResponse=False ):
@@ -2542,15 +2541,15 @@
                 main.cleanup()
                 main.exit()
             return None
+        except pexpect.TIMEOUT:
+            main.log.error( self.name + ": ONOS timeout" )
+            return None
         except Exception:
             main.log.exception( self.name + ": Uncaught exception!" )
             if not noExit:
                 main.cleanup()
                 main.exit()
             return None
-        except pexpect.TIMEOUT:
-            main.log.error( self.name + ": ONOS timeout" )
-            return None
 
     def getTotalIntentsNum( self, timeout=60, noExit = False ):
         """
@@ -2564,8 +2563,8 @@
         try:
             cmd = "summary -j"
             response = self.sendline( cmd, timeout=timeout, noExit=noExit )
-            if response == None:
-                return  -1
+            if response is None:
+                return -1
             response = json.loads( response )
             return int( response.get("intents") )
         except ( TypeError, ValueError ):
@@ -2880,7 +2879,7 @@
             return main.ERROR
         try:
             topology = self.getTopology( self.topology() )
-            if topology == {} or topology == None or summary == {} or summary == None:
+            if topology == {} or topology is None or summary == {} or summary is None:
                 return main.ERROR
             output = ""
             # Is the number of switches is what we expected
@@ -4167,7 +4166,7 @@
                     containsCheck = main.FALSE
                     match = matchFalse
                 else:
-                    main.log.error( self.name + " setTestGet did not match " +\
+                    main.log.error( self.name + " setTestGet did not match " +
                                     "expected output" )
                     main.log.debug( self.name + " expected: " + pattern )
                     main.log.debug( self.name + " actual: " + repr( output ) )
@@ -4253,7 +4252,6 @@
             None on error
         """
         try:
-            counters = {}
             cmdStr = "counters"
             if jsonFormat:
                 cmdStr += " -j"
@@ -4871,7 +4869,7 @@
             present in the resoponse. Otherwise, returns main.FALSE
         '''
         try:
-            cmd =  "null-link null:{} null:{} {}".format( begin, end, state )
+            cmd = "null-link null:{} null:{} {}".format( begin, end, state )
             response = self.sendline( cmd, showResponse=showResponse, timeout=timeout )
             assert response is not None, "Error in sendline"
             assert "Command not found:" not in response, response
@@ -4911,7 +4909,7 @@
         try:
             state = state.lower()
             assert state == 'enable' or state == 'disable', "Unknown state"
-            cmd =  "portstate {} {} {}".format( dpid, port, state )
+            cmd = "portstate {} {} {}".format( dpid, port, state )
             response = self.sendline( cmd, showResponse=True )
             assert response is not None, "Error in sendline"
             assert "Command not found:" not in response, response
@@ -4945,7 +4943,7 @@
         Level defaults to INFO
         """
         try:
-            self.handle.sendline( "log:set %s %s" %( level, app ) )
+            self.handle.sendline( "log:set %s %s" % ( level, app ) )
             self.handle.expect( "onos>" )
 
             response = self.handle.before
@@ -4999,24 +4997,24 @@
                 nodeA = link[ 'src' ][ 'device' ]
                 nodeB = link[ 'dst' ][ 'device' ]
                 assert idToDevice[ nodeA ][ 'available' ] and idToDevice[ nodeB ][ 'available' ]
-                if not nodeA in graphDict.keys():
-                    graphDict[ nodeA ] = { 'edges':{},
-                                           'dpid':idToDevice[ nodeA ][ 'id' ][3:],
-                                           'type':idToDevice[ nodeA ][ 'type' ],
-                                           'available':idToDevice[ nodeA ][ 'available' ],
-                                           'role':idToDevice[ nodeA ][ 'role' ],
-                                           'mfr':idToDevice[ nodeA ][ 'mfr' ],
-                                           'hw':idToDevice[ nodeA ][ 'hw' ],
-                                           'sw':idToDevice[ nodeA ][ 'sw' ],
-                                           'serial':idToDevice[ nodeA ][ 'serial' ],
-                                           'chassisId':idToDevice[ nodeA ][ 'chassisId' ],
-                                           'annotations':idToDevice[ nodeA ][ 'annotations' ]}
+                if nodeA not in graphDict.keys():
+                    graphDict[ nodeA ] = { 'edges': {},
+                                           'dpid': idToDevice[ nodeA ][ 'id' ][3:],
+                                           'type': idToDevice[ nodeA ][ 'type' ],
+                                           'available': idToDevice[ nodeA ][ 'available' ],
+                                           'role': idToDevice[ nodeA ][ 'role' ],
+                                           'mfr': idToDevice[ nodeA ][ 'mfr' ],
+                                           'hw': idToDevice[ nodeA ][ 'hw' ],
+                                           'sw': idToDevice[ nodeA ][ 'sw' ],
+                                           'serial': idToDevice[ nodeA ][ 'serial' ],
+                                           'chassisId': idToDevice[ nodeA ][ 'chassisId' ],
+                                           'annotations': idToDevice[ nodeA ][ 'annotations' ]}
                 else:
                     # Assert nodeB is not connected to any current links of nodeA
                     assert nodeB not in graphDict[ nodeA ][ 'edges' ].keys()
-                graphDict[ nodeA ][ 'edges' ][ nodeB ] = { 'port':link[ 'src' ][ 'port' ],
-                                                           'type':link[ 'type' ],
-                                                           'state':link[ 'state' ] }
+                graphDict[ nodeA ][ 'edges' ][ nodeB ] = { 'port': link[ 'src' ][ 'port' ],
+                                                           'type': link[ 'type' ],
+                                                           'state': link[ 'state' ] }
             return graphDict
         except ( TypeError, ValueError ):
             main.log.exception( self.name + ": Object not as expected" )
@@ -5097,7 +5095,7 @@
         """
         try:
             assert type( searchTerm ) is str
-            #Build the log paths string
+            # Build the log paths string
             logPath = '/opt/onos/log/karaf.log.'
             logPaths = '/opt/onos/log/karaf.log'
             for i in range( 1, logNum ):
@@ -5463,7 +5461,7 @@
         try:
             cmdStr = "interfaces"
             if jsonFormat:
-                #raise NotImplementedError
+                raise NotImplementedError
                 cmdStr += " -j"
             handle = self.sendline( cmdStr )
             assert handle is not None, "Error in sendline"
@@ -5501,10 +5499,7 @@
                 and the splitTerm_after is "x"
 
             others:
-
-                plz look at the "logsearch" Function in onosclidriver.py
-
-
+                Please look at the "logsearch" Function in onosclidriver.py
         '''
         if logNum < 0:
             main.log.error("Get wrong log number ")
@@ -5526,3 +5521,240 @@
         except AssertionError:
             main.log.warn( "Search Term Not Found " )
             return main.ERROR
+
+    def workQueueAdd( self, queueName, value ):
+        """
+        CLI command to add a string to the specified Work Queue.
+        This function uses the distributed primitives test app, which
+        gives some cli access to distributed primitives for testing
+        purposes only.
+
+        Required arguments:
+            queueName - The name of the queue to add to
+            value - The value to add to the queue
+        returns:
+            main.TRUE on success, main.FALSE on failure and
+            main.ERROR on error.
+        """
+        try:
+            queueName = str( queueName )
+            value = str( value )
+            prefix = "work-queue-test"
+            operation = "add"
+            cmdStr = " ".join( [ prefix, queueName, operation, value ] )
+            output = self.distPrimitivesSend( cmdStr )
+            if "Invalid operation name" in output:
+                main.log.warn( output )
+                return main.ERROR
+            elif "Done" in output:
+                return main.TRUE
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def workQueueAddMultiple( self, queueName, value1, value2 ):
+        """
+        CLI command to add two strings to the specified Work Queue.
+        This function uses the distributed primitives test app, which
+        gives some cli access to distributed primitives for testing
+        purposes only.
+
+        Required arguments:
+            queueName - The name of the queue to add to
+            value1 - The first value to add to the queue
+            value2 - The second value to add to the queue
+        returns:
+            main.TRUE on success, main.FALSE on failure and
+            main.ERROR on error.
+        """
+        try:
+            queueName = str( queueName )
+            value1 = str( value1 )
+            value2 = str( value2 )
+            prefix = "work-queue-test"
+            operation = "addMultiple"
+            cmdStr = " ".join( [ prefix, queueName, operation, value1, value2 ] )
+            output = self.distPrimitivesSend( cmdStr )
+            if "Invalid operation name" in output:
+                main.log.warn( output )
+                return main.ERROR
+            elif "Done" in output:
+                return main.TRUE
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def workQueueTakeAndComplete( self, queueName, number=1 ):
+        """
+        CLI command to take a value from the specified Work Queue and compelte it.
+        This function uses the distributed primitives test app, which
+        gives some cli access to distributed primitives for testing
+        purposes only.
+
+        Required arguments:
+            queueName - The name of the queue to add to
+            number - The number of items to take and complete
+        returns:
+            main.TRUE on success, main.FALSE on failure and
+            main.ERROR on error.
+        """
+        try:
+            queueName = str( queueName )
+            number = str( int( number ) )
+            prefix = "work-queue-test"
+            operation = "takeAndComplete"
+            cmdStr = " ".join( [ prefix, queueName, operation, number ] )
+            output = self.distPrimitivesSend( cmdStr )
+            if "Invalid operation name" in output:
+                main.log.warn( output )
+                return main.ERROR
+            elif "Done" in output:
+                return main.TRUE
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def workQueueDestroy( self, queueName ):
+        """
+        CLI command to destroy the specified Work Queue.
+        This function uses the distributed primitives test app, which
+        gives some cli access to distributed primitives for testing
+        purposes only.
+
+        Required arguments:
+            queueName - The name of the queue to add to
+        returns:
+            main.TRUE on success, main.FALSE on failure and
+            main.ERROR on error.
+        """
+        try:
+            queueName = str( queueName )
+            prefix = "work-queue-test"
+            operation = "destroy"
+            cmdStr = " ".join( [ prefix, queueName, operation ] )
+            output = self.distPrimitivesSend( cmdStr )
+            if "Invalid operation name" in output:
+                main.log.warn( output )
+                return main.ERROR
+            return main.TRUE
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def workQueueTotalPending( self, queueName ):
+        """
+        CLI command to get the Total Pending items of the specified Work Queue.
+        This function uses the distributed primitives test app, which
+        gives some cli access to distributed primitives for testing
+        purposes only.
+
+        Required arguments:
+            queueName - The name of the queue to add to
+        returns:
+            The number of Pending items in the specified work queue or
+            None on error
+        """
+        try:
+            queueName = str( queueName )
+            prefix = "work-queue-test"
+            operation = "totalPending"
+            cmdStr = " ".join( [ prefix, queueName, operation ] )
+            output = self.distPrimitivesSend( cmdStr )
+            pattern = r'\d+'
+            if "Invalid operation name" in output:
+                main.log.warn( output )
+                return None
+            else:
+                match = re.search( pattern, output )
+                return match.group(0)
+        except ( AttributeError, TypeError ):
+            main.log.exception( self.name + ": Object not as expected; " + str( output ) )
+            return None
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def workQueueTotalCompleted( self, queueName ):
+        """
+        CLI command to get the Total Completed items of the specified Work Queue.
+        This function uses the distributed primitives test app, which
+        gives some cli access to distributed primitives for testing
+        purposes only.
+
+        Required arguments:
+            queueName - The name of the queue to add to
+        returns:
+            The number of complete items in the specified work queue or
+            None on error
+        """
+        try:
+            queueName = str( queueName )
+            prefix = "work-queue-test"
+            operation = "totalCompleted"
+            cmdStr = " ".join( [ prefix, queueName, operation ] )
+            output = self.distPrimitivesSend( cmdStr )
+            pattern = r'\d+'
+            if "Invalid operation name" in output:
+                main.log.warn( output )
+                return None
+            else:
+                match = re.search( pattern, output )
+                return match.group(0)
+        except ( AttributeError, TypeError ):
+            main.log.exception( self.name + ": Object not as expected; " + str( output ) )
+            return None
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def workQueueTotalInProgress( self, queueName ):
+        """
+        CLI command to get the Total In Progress items of the specified Work Queue.
+        This function uses the distributed primitives test app, which
+        gives some cli access to distributed primitives for testing
+        purposes only.
+
+        Required arguments:
+            queueName - The name of the queue to add to
+        returns:
+            The number of In Progress items in the specified work queue or
+            None on error
+        """
+        try:
+            queueName = str( queueName )
+            prefix = "work-queue-test"
+            operation = "totalInProgress"
+            cmdStr = " ".join( [ prefix, queueName, operation ] )
+            output = self.distPrimitivesSend( cmdStr )
+            pattern = r'\d+'
+            if "Invalid operation name" in output:
+                main.log.warn( output )
+                return None
+            else:
+                match = re.search( pattern, output )
+                return match.group(0)
+        except ( AttributeError, TypeError ):
+            main.log.exception( self.name + ": Object not as expected; " + str( output ) )
+            return None
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index b6c6f26..7df3e51 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -109,7 +109,7 @@
     def consistentLeaderboards( self, nodes ):
         TOPIC = 'org.onosproject.election'
         # FIXME: use threads
-        #FIXME: should we retry outside the function?
+        # FIXME: should we retry outside the function?
         for n in range( 5 ):  # Retry in case election is still happening
             leaderList = []
             # Get all leaderboards
@@ -160,11 +160,73 @@
             results = results and currentResult
         return results
 
+    def workQueueStatsCheck( self, workQueueName, completed, inProgress, pending ):
+        # Completed
+        threads = []
+        completedValues = []
+        for i in main.activeNodes:
+            t = main.Thread( target=main.CLIs[i].workQueueTotalCompleted,
+                             name="WorkQueueCompleted-" + str( i ),
+                             args=[ workQueueName ] )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            completedValues.append( int( t.result ) )
+        # Check the results
+        completedResults = [ x == completed for x in completedValues ]
+        completedResult = all( completedResults )
+        if not completedResult:
+            main.log.warn( "Expected Work Queue {} to have {} completed, found {}".format(
+                workQueueName, completed, completedValues ) )
+
+        # In Progress
+        threads = []
+        inProgressValues = []
+        for i in main.activeNodes:
+            t = main.Thread( target=main.CLIs[i].workQueueTotalInProgress,
+                             name="WorkQueueInProgress-" + str( i ),
+                             args=[ workQueueName ] )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            inProgressValues.append( int( t.result ) )
+        # Check the results
+        inProgressResults = [ x == inProgress for x in inProgressValues ]
+        inProgressResult = all( inProgressResults )
+        if not inProgressResult:
+            main.log.warn( "Expected Work Queue {} to have {} inProgress, found {}".format(
+                workQueueName, inProgress, inProgressValues ) )
+
+        # Pending
+        threads = []
+        pendingValues = []
+        for i in main.activeNodes:
+            t = main.Thread( target=main.CLIs[i].workQueueTotalPending,
+                             name="WorkQueuePending-" + str( i ),
+                             args=[ workQueueName ] )
+            threads.append( t )
+            t.start()
+
+        for t in threads:
+            t.join()
+            pendingValues.append( int( t.result ) )
+        # Check the results
+        pendingResults = [ x == pending for x in pendingValues ]
+        pendingResult = all( pendingResults )
+        if not pendingResult:
+            main.log.warn( "Expected Work Queue {} to have {} pending, found {}".format(
+                workQueueName, pending, pendingValues ) )
+        return completedResult and inProgressResult and pendingResult
+
     def CASE17( self, main ):
         """
         Check for basic functionality with distributed primitives
         """
-        #TODO: Clean this up so it's not just a cut/paste from the test
+        # TODO: Clean this up so it's not just a cut/paste from the test
         try:
             # Make sure variables are defined/set
             assert main.numCtrls, "main.numCtrls not defined"
@@ -188,8 +250,12 @@
             addValue = "a"
             addAllValue = "a b c d e f"
             retainValue = "c d e f"
-            valueName = "TestOn-Value"
+            valueName = "TestON-Value"
             valueValue = None
+            workQueueName = "TestON-Queue"
+            workQueueCompleted = 0
+            workQueueInProgress = 0
+            workQueuePending = 0
 
             description = "Check for basic functionality with distributed " +\
                           "primitives"
@@ -1522,5 +1588,123 @@
                                      onfail="Error getting atomic Value " +
                                             str( valueValue ) + ", found: " +
                                             str( getValues ) )
+
+            # WORK QUEUES
+            main.step( "Work Queue add()" )
+            threads = []
+            i = main.activeNodes[0]
+            addResult = main.CLIs[i].workQueueAdd( workQueueName, 'foo' )
+            workQueuePending += 1
+            main.log.debug( addResult )
+            # Check the results
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=addResult,
+                                     onpass="Work Queue add successful",
+                                     onfail="Error adding to Work Queue" )
+
+            main.step( "Check the work queue stats" )
+            statsResults = self.workQueueStatsCheck( workQueueName,
+                                                     workQueueCompleted,
+                                                     workQueueInProgress,
+                                                     workQueuePending )
+            utilities.assert_equals( expect=True,
+                                     actual=statsResults,
+                                     onpass="Work Queue stats correct",
+                                     onfail="Work Queue stats incorrect " )
+
+            main.step( "Work Queue addMultiple()" )
+            threads = []
+            i = main.activeNodes[0]
+            addMultipleResult = main.CLIs[i].workQueueAddMultiple( workQueueName, 'bar', 'baz' )
+            workQueuePending += 2
+            main.log.debug( addMultipleResult )
+            # Check the results
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=addMultipleResult,
+                                     onpass="Work Queue add multiple successful",
+                                     onfail="Error adding multiple items to Work Queue" )
+
+            main.step( "Check the work queue stats" )
+            statsResults = self.workQueueStatsCheck( workQueueName,
+                                                     workQueueCompleted,
+                                                     workQueueInProgress,
+                                                     workQueuePending )
+            utilities.assert_equals( expect=True,
+                                     actual=statsResults,
+                                     onpass="Work Queue stats correct",
+                                     onfail="Work Queue stats incorrect " )
+
+            main.step( "Work Queue takeAndComplete() 1" )
+            threads = []
+            i = main.activeNodes[0]
+            number = 1
+            take1Result = main.CLIs[i].workQueueTakeAndComplete( workQueueName, number )
+            workQueuePending -= number
+            workQueueCompleted += number
+            main.log.debug( take1Result )
+            # Check the results
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=take1Result,
+                                     onpass="Work Queue takeAndComplete 1 successful",
+                                     onfail="Error taking 1 from Work Queue" )
+
+            main.step( "Check the work queue stats" )
+            statsResults = self.workQueueStatsCheck( workQueueName,
+                                                     workQueueCompleted,
+                                                     workQueueInProgress,
+                                                     workQueuePending )
+            utilities.assert_equals( expect=True,
+                                     actual=statsResults,
+                                     onpass="Work Queue stats correct",
+                                     onfail="Work Queue stats incorrect " )
+
+            main.step( "Work Queue takeAndComplete() 2" )
+            threads = []
+            i = main.activeNodes[0]
+            number = 2
+            take2Result = main.CLIs[i].workQueueTakeAndComplete( workQueueName, number )
+            workQueuePending -= number
+            workQueueCompleted += number
+            main.log.debug( take2Result )
+            # Check the results
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=take2Result,
+                                     onpass="Work Queue takeAndComplete 2 successful",
+                                     onfail="Error taking 2 from Work Queue" )
+
+            main.step( "Check the work queue stats" )
+            statsResults = self.workQueueStatsCheck( workQueueName,
+                                                     workQueueCompleted,
+                                                     workQueueInProgress,
+                                                     workQueuePending )
+            utilities.assert_equals( expect=True,
+                                     actual=statsResults,
+                                     onpass="Work Queue stats correct",
+                                     onfail="Work Queue stats incorrect " )
+
+            main.step( "Work Queue destroy()" )
+            valueValue = None
+            threads = []
+            i = main.activeNodes[0]
+            destroyResult = main.CLIs[i].workQueueDestroy( workQueueName )
+            workQueueCompleted = 0
+            workQueueInProgress = 0
+            workQueuePending = 0
+            main.log.debug( destroyResult )
+            # Check the results
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=destroyResult,
+                                     onpass="Work Queue destroy successful",
+                                     onfail="Error destroying Work Queue" )
+
+            main.step( "Check the work queue stats" )
+            statsResults = self.workQueueStatsCheck( workQueueName,
+                                                     workQueueCompleted,
+                                                     workQueueInProgress,
+                                                     workQueuePending )
+            utilities.assert_equals( expect=True,
+                                     actual=statsResults,
+                                     onpass="Work Queue stats correct",
+                                     onfail="Work Queue stats incorrect " )
         except Exception as e:
             main.log.error( "Exception: " + str( e ) )