Add tests for atomic value to HA suite

- Add driver functions
- Add to HA tests
- Create a helper function to handle errors with the distributed
  primatives test cli
- Part of [ONOS-2451]
- Make Distributed primitives test case a function for the HA suite

Change-Id: Ieabe55ccee6c10e52188061126f599e8bb1dec30
diff --git a/TestON/tests/HA/HAsanity/HAsanity.py b/TestON/tests/HA/HAsanity/HAsanity.py
index 32b7e14..2b2ea08 100644
--- a/TestON/tests/HA/HAsanity/HAsanity.py
+++ b/TestON/tests/HA/HAsanity/HAsanity.py
@@ -2976,7 +2976,7 @@
         # verify leader didn't just change
         # Get new leaders and candidates
         reRunLeaders = []
-        time.sleep( 5 ) # Paremterize
+        time.sleep( 5 )  # Paremterize
         positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
 
         # Check that the re-elected node is last on the candidate List
@@ -3005,14 +3005,10 @@
         assert main.nodes, "main.nodes not defined"
 
         # Variables for the distributed primitives tests
-        global pCounterName
-        global pCounterValue
-        global onosSet
-        global onosSetName
-        pCounterName = "TestON-Partitions"
-        pCounterValue = 0
-        onosSet = set([])
-        onosSetName = "TestON-set"
+        main.pCounterName = "TestON-Partitions"
+        main.pCounterValue = 0
+        main.onosSet = set([])
+        main.onosSetName = "TestON-set"
 
         description = "Install Primitives app"
         main.case( description )
@@ -3030,1153 +3026,4 @@
         """
         Check for basic functionality with distributed primitives
         """
-        # Make sure variables are defined/set
-        assert main.numCtrls, "main.numCtrls not defined"
-        assert main, "main not defined"
-        assert utilities.assert_equals, "utilities.assert_equals not defined"
-        assert main.CLIs, "main.CLIs not defined"
-        assert main.nodes, "main.nodes not defined"
-        assert pCounterName, "pCounterName not defined"
-        assert onosSetName, "onosSetName not defined"
-        # NOTE: assert fails if value is 0/None/Empty/False
-        try:
-            pCounterValue
-        except NameError:
-            main.log.error( "pCounterValue not defined, setting to 0" )
-            pCounterValue = 0
-        try:
-            onosSet
-        except NameError:
-            main.log.error( "onosSet not defined, setting to empty Set" )
-            onosSet = set([])
-        # Variables for the distributed primitives tests. These are local only
-        addValue = "a"
-        addAllValue = "a b c d e f"
-        retainValue = "c d e f"
-
-        description = "Check for basic functionality with distributed " +\
-                      "primitives"
-        main.case( description )
-        main.caseExplanation = "Test the methods of the distributed " +\
-                                "primitives (counters and sets) throught the cli"
-        # DISTRIBUTED ATOMIC COUNTERS
-        # Partitioned counters
-        main.step( "Increment then get a default counter on each node" )
-        pCounters = []
-        threads = []
-        addedPValues = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
-                             name="counterAddAndGet-" + str( i ),
-                             args=[ pCounterName ] )
-            pCounterValue += 1
-            addedPValues.append( pCounterValue )
-            threads.append( t )
-            t.start()
-
-        for t in threads:
-            t.join()
-            pCounters.append( t.result )
-        # Check that counter incremented numController times
-        pCounterResults = True
-        for i in addedPValues:
-            tmpResult = i in pCounters
-            pCounterResults = pCounterResults and tmpResult
-            if not tmpResult:
-                main.log.error( str( i ) + " is not in partitioned "
-                                "counter incremented results" )
-        utilities.assert_equals( expect=True,
-                                 actual=pCounterResults,
-                                 onpass="Default counter incremented",
-                                 onfail="Error incrementing default" +
-                                        " counter" )
-
-        main.step( "Get then Increment a default counter on each node" )
-        pCounters = []
-        threads = []
-        addedPValues = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
-                             name="counterGetAndAdd-" + str( i ),
-                             args=[ pCounterName ] )
-            addedPValues.append( pCounterValue )
-            pCounterValue += 1
-            threads.append( t )
-            t.start()
-
-        for t in threads:
-            t.join()
-            pCounters.append( t.result )
-        # Check that counter incremented numController times
-        pCounterResults = True
-        for i in addedPValues:
-            tmpResult = i in pCounters
-            pCounterResults = pCounterResults and tmpResult
-            if not tmpResult:
-                main.log.error( str( i ) + " is not in partitioned "
-                                "counter incremented results" )
-        utilities.assert_equals( expect=True,
-                                 actual=pCounterResults,
-                                 onpass="Default counter incremented",
-                                 onfail="Error incrementing default" +
-                                        " counter" )
-
-        main.step( "Counters we added have the correct values" )
-        incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=incrementCheck,
-                                 onpass="Added counters are correct",
-                                 onfail="Added counters are incorrect" )
-
-        main.step( "Add -8 to then get a default counter on each node" )
-        pCounters = []
-        threads = []
-        addedPValues = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
-                             name="counterIncrement-" + str( i ),
-                             args=[ pCounterName ],
-                             kwargs={ "delta": -8 } )
-            pCounterValue += -8
-            addedPValues.append( pCounterValue )
-            threads.append( t )
-            t.start()
-
-        for t in threads:
-            t.join()
-            pCounters.append( t.result )
-        # Check that counter incremented numController times
-        pCounterResults = True
-        for i in addedPValues:
-            tmpResult = i in pCounters
-            pCounterResults = pCounterResults and tmpResult
-            if not tmpResult:
-                main.log.error( str( i ) + " is not in partitioned "
-                                "counter incremented results" )
-        utilities.assert_equals( expect=True,
-                                 actual=pCounterResults,
-                                 onpass="Default counter incremented",
-                                 onfail="Error incrementing default" +
-                                        " counter" )
-
-        main.step( "Add 5 to then get a default counter on each node" )
-        pCounters = []
-        threads = []
-        addedPValues = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
-                             name="counterIncrement-" + str( i ),
-                             args=[ pCounterName ],
-                             kwargs={ "delta": 5 } )
-            pCounterValue += 5
-            addedPValues.append( pCounterValue )
-            threads.append( t )
-            t.start()
-
-        for t in threads:
-            t.join()
-            pCounters.append( t.result )
-        # Check that counter incremented numController times
-        pCounterResults = True
-        for i in addedPValues:
-            tmpResult = i in pCounters
-            pCounterResults = pCounterResults and tmpResult
-            if not tmpResult:
-                main.log.error( str( i ) + " is not in partitioned "
-                                "counter incremented results" )
-        utilities.assert_equals( expect=True,
-                                 actual=pCounterResults,
-                                 onpass="Default counter incremented",
-                                 onfail="Error incrementing default" +
-                                        " counter" )
-
-        main.step( "Get then add 5 to a default counter on each node" )
-        pCounters = []
-        threads = []
-        addedPValues = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
-                             name="counterIncrement-" + str( i ),
-                             args=[ pCounterName ],
-                             kwargs={ "delta": 5 } )
-            addedPValues.append( pCounterValue )
-            pCounterValue += 5
-            threads.append( t )
-            t.start()
-
-        for t in threads:
-            t.join()
-            pCounters.append( t.result )
-        # Check that counter incremented numController times
-        pCounterResults = True
-        for i in addedPValues:
-            tmpResult = i in pCounters
-            pCounterResults = pCounterResults and tmpResult
-            if not tmpResult:
-                main.log.error( str( i ) + " is not in partitioned "
-                                "counter incremented results" )
-        utilities.assert_equals( expect=True,
-                                 actual=pCounterResults,
-                                 onpass="Default counter incremented",
-                                 onfail="Error incrementing default" +
-                                        " counter" )
-
-        main.step( "Counters we added have the correct values" )
-        incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=incrementCheck,
-                                 onpass="Added counters are correct",
-                                 onfail="Added counters are incorrect" )
-
-        # DISTRIBUTED SETS
-        main.step( "Distributed Set get" )
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node +
-                                        " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node +
-                                    " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=getResults,
-                                 onpass="Set elements are correct",
-                                 onfail="Set elements are incorrect" )
-
-        main.step( "Distributed Set size" )
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node +
-                                " expected a size of " + str( size ) +
-                                " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=sizeResults,
-                                 onpass="Set sizes are correct",
-                                 onfail="Set sizes are incorrect" )
-
-        main.step( "Distributed Set add()" )
-        onosSet.add( addValue )
-        addResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestAdd,
-                             name="setTestAdd-" + str( i ),
-                             args=[ onosSetName, addValue ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            addResponses.append( t.result )
-
-        # main.TRUE = successfully changed the set
-        # main.FALSE = action resulted in no change in set
-        # main.ERROR - Some error in executing the function
-        addResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if addResponses[ i ] == main.TRUE:
-                # All is well
-                pass
-            elif addResponses[ i ] == main.FALSE:
-                # Already in set, probably fine
-                pass
-            elif addResponses[ i ] == main.ERROR:
-                # Error in execution
-                addResults = main.FALSE
-            else:
-                # unexpected result
-                addResults = main.FALSE
-        if addResults != main.TRUE:
-            main.log.error( "Error executing set add" )
-
-        # Check if set is still correct
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node + " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node + " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node +
-                                " expected a size of " + str( size ) +
-                                " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        addResults = addResults and getResults and sizeResults
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=addResults,
-                                 onpass="Set add correct",
-                                 onfail="Set add was incorrect" )
-
-        main.step( "Distributed Set addAll()" )
-        onosSet.update( addAllValue.split() )
-        addResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestAdd,
-                             name="setTestAddAll-" + str( i ),
-                             args=[ onosSetName, addAllValue ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            addResponses.append( t.result )
-
-        # main.TRUE = successfully changed the set
-        # main.FALSE = action resulted in no change in set
-        # main.ERROR - Some error in executing the function
-        addAllResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if addResponses[ i ] == main.TRUE:
-                # All is well
-                pass
-            elif addResponses[ i ] == main.FALSE:
-                # Already in set, probably fine
-                pass
-            elif addResponses[ i ] == main.ERROR:
-                # Error in execution
-                addAllResults = main.FALSE
-            else:
-                # unexpected result
-                addAllResults = main.FALSE
-        if addAllResults != main.TRUE:
-            main.log.error( "Error executing set addAll" )
-
-        # Check if set is still correct
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node +
-                                        " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node +
-                                    " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node +
-                                " expected a size of " + str( size ) +
-                                " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        addAllResults = addAllResults and getResults and sizeResults
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=addAllResults,
-                                 onpass="Set addAll correct",
-                                 onfail="Set addAll was incorrect" )
-
-        main.step( "Distributed Set contains()" )
-        containsResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setContains-" + str( i ),
-                             args=[ onosSetName ],
-                             kwargs={ "values": addValue } )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            # NOTE: This is the tuple
-            containsResponses.append( t.result )
-
-        containsResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if containsResponses[ i ] == main.ERROR:
-                containsResults = main.FALSE
-            else:
-                containsResults = containsResults and\
-                                  containsResponses[ i ][ 1 ]
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=containsResults,
-                                 onpass="Set contains is functional",
-                                 onfail="Set contains failed" )
-
-        main.step( "Distributed Set containsAll()" )
-        containsAllResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setContainsAll-" + str( i ),
-                             args=[ onosSetName ],
-                             kwargs={ "values": addAllValue } )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            # NOTE: This is the tuple
-            containsAllResponses.append( t.result )
-
-        containsAllResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if containsResponses[ i ] == main.ERROR:
-                containsResults = main.FALSE
-            else:
-                containsResults = containsResults and\
-                                  containsResponses[ i ][ 1 ]
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=containsAllResults,
-                                 onpass="Set containsAll is functional",
-                                 onfail="Set containsAll failed" )
-
-        main.step( "Distributed Set remove()" )
-        onosSet.remove( addValue )
-        removeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestRemove,
-                             name="setTestRemove-" + str( i ),
-                             args=[ onosSetName, addValue ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            removeResponses.append( t.result )
-
-        # main.TRUE = successfully changed the set
-        # main.FALSE = action resulted in no change in set
-        # main.ERROR - Some error in executing the function
-        removeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if removeResponses[ i ] == main.TRUE:
-                # All is well
-                pass
-            elif removeResponses[ i ] == main.FALSE:
-                # not in set, probably fine
-                pass
-            elif removeResponses[ i ] == main.ERROR:
-                # Error in execution
-                removeResults = main.FALSE
-            else:
-                # unexpected result
-                removeResults = main.FALSE
-        if removeResults != main.TRUE:
-            main.log.error( "Error executing set remove" )
-
-        # Check if set is still correct
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node +
-                                        " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node +
-                                    " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node +
-                                " expected a size of " + str( size ) +
-                                " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        removeResults = removeResults and getResults and sizeResults
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=removeResults,
-                                 onpass="Set remove correct",
-                                 onfail="Set remove was incorrect" )
-
-        main.step( "Distributed Set removeAll()" )
-        onosSet.difference_update( addAllValue.split() )
-        removeAllResponses = []
-        threads = []
-        try:
-            for i in main.activeNodes:
-                t = main.Thread( target=main.CLIs[i].setTestRemove,
-                                 name="setTestRemoveAll-" + str( i ),
-                                 args=[ onosSetName, addAllValue ] )
-                threads.append( t )
-                t.start()
-            for t in threads:
-                t.join()
-                removeAllResponses.append( t.result )
-        except Exception, e:
-            main.log.exception(e)
-
-        # main.TRUE = successfully changed the set
-        # main.FALSE = action resulted in no change in set
-        # main.ERROR - Some error in executing the function
-        removeAllResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if removeAllResponses[ i ] == main.TRUE:
-                # All is well
-                pass
-            elif removeAllResponses[ i ] == main.FALSE:
-                # not in set, probably fine
-                pass
-            elif removeAllResponses[ i ] == main.ERROR:
-                # Error in execution
-                removeAllResults = main.FALSE
-            else:
-                # unexpected result
-                removeAllResults = main.FALSE
-        if removeAllResults != main.TRUE:
-            main.log.error( "Error executing set removeAll" )
-
-        # Check if set is still correct
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node +
-                                        " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node +
-                                    " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node +
-                                " expected a size of " + str( size ) +
-                                " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        removeAllResults = removeAllResults and getResults and sizeResults
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=removeAllResults,
-                                 onpass="Set removeAll correct",
-                                 onfail="Set removeAll was incorrect" )
-
-        main.step( "Distributed Set addAll()" )
-        onosSet.update( addAllValue.split() )
-        addResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestAdd,
-                             name="setTestAddAll-" + str( i ),
-                             args=[ onosSetName, addAllValue ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            addResponses.append( t.result )
-
-        # main.TRUE = successfully changed the set
-        # main.FALSE = action resulted in no change in set
-        # main.ERROR - Some error in executing the function
-        addAllResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if addResponses[ i ] == main.TRUE:
-                # All is well
-                pass
-            elif addResponses[ i ] == main.FALSE:
-                # Already in set, probably fine
-                pass
-            elif addResponses[ i ] == main.ERROR:
-                # Error in execution
-                addAllResults = main.FALSE
-            else:
-                # unexpected result
-                addAllResults = main.FALSE
-        if addAllResults != main.TRUE:
-            main.log.error( "Error executing set addAll" )
-
-        # Check if set is still correct
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node +
-                                        " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node +
-                                    " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node +
-                                " expected a size of " + str( size ) +
-                                " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        addAllResults = addAllResults and getResults and sizeResults
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=addAllResults,
-                                 onpass="Set addAll correct",
-                                 onfail="Set addAll was incorrect" )
-
-        main.step( "Distributed Set clear()" )
-        onosSet.clear()
-        clearResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestRemove,
-                             name="setTestClear-" + str( i ),
-                             args=[ onosSetName, " "],  # Values doesn't matter
-                             kwargs={ "clear": True } )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            clearResponses.append( t.result )
-
-        # main.TRUE = successfully changed the set
-        # main.FALSE = action resulted in no change in set
-        # main.ERROR - Some error in executing the function
-        clearResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if clearResponses[ i ] == main.TRUE:
-                # All is well
-                pass
-            elif clearResponses[ i ] == main.FALSE:
-                # Nothing set, probably fine
-                pass
-            elif clearResponses[ i ] == main.ERROR:
-                # Error in execution
-                clearResults = main.FALSE
-            else:
-                # unexpected result
-                clearResults = main.FALSE
-        if clearResults != main.TRUE:
-            main.log.error( "Error executing set clear" )
-
-        # Check if set is still correct
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node +
-                                        " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node +
-                                    " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node +
-                                " expected a size of " + str( size ) +
-                                " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        clearResults = clearResults and getResults and sizeResults
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=clearResults,
-                                 onpass="Set clear correct",
-                                 onfail="Set clear was incorrect" )
-
-        main.step( "Distributed Set addAll()" )
-        onosSet.update( addAllValue.split() )
-        addResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestAdd,
-                             name="setTestAddAll-" + str( i ),
-                             args=[ onosSetName, addAllValue ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            addResponses.append( t.result )
-
-        # main.TRUE = successfully changed the set
-        # main.FALSE = action resulted in no change in set
-        # main.ERROR - Some error in executing the function
-        addAllResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if addResponses[ i ] == main.TRUE:
-                # All is well
-                pass
-            elif addResponses[ i ] == main.FALSE:
-                # Already in set, probably fine
-                pass
-            elif addResponses[ i ] == main.ERROR:
-                # Error in execution
-                addAllResults = main.FALSE
-            else:
-                # unexpected result
-                addAllResults = main.FALSE
-        if addAllResults != main.TRUE:
-            main.log.error( "Error executing set addAll" )
-
-        # Check if set is still correct
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node +
-                                        " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node +
-                                    " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node +
-                                " expected a size of " + str( size ) +
-                                " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        addAllResults = addAllResults and getResults and sizeResults
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=addAllResults,
-                                 onpass="Set addAll correct",
-                                 onfail="Set addAll was incorrect" )
-
-        main.step( "Distributed Set retain()" )
-        onosSet.intersection_update( retainValue.split() )
-        retainResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestRemove,
-                             name="setTestRetain-" + str( i ),
-                             args=[ onosSetName, retainValue ],
-                             kwargs={ "retain": True } )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            retainResponses.append( t.result )
-
-        # main.TRUE = successfully changed the set
-        # main.FALSE = action resulted in no change in set
-        # main.ERROR - Some error in executing the function
-        retainResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            if retainResponses[ i ] == main.TRUE:
-                # All is well
-                pass
-            elif retainResponses[ i ] == main.FALSE:
-                # Already in set, probably fine
-                pass
-            elif retainResponses[ i ] == main.ERROR:
-                # Error in execution
-                retainResults = main.FALSE
-            else:
-                # unexpected result
-                retainResults = main.FALSE
-        if retainResults != main.TRUE:
-            main.log.error( "Error executing set retain" )
-
-        # Check if set is still correct
-        size = len( onosSet )
-        getResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestGet,
-                             name="setTestGet-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            getResponses.append( t.result )
-        getResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if isinstance( getResponses[ i ], list):
-                current = set( getResponses[ i ] )
-                if len( current ) == len( getResponses[ i ] ):
-                    # no repeats
-                    if onosSet != current:
-                        main.log.error( "ONOS" + node +
-                                        " has incorrect view" +
-                                        " of set " + onosSetName + ":\n" +
-                                        str( getResponses[ i ] ) )
-                        main.log.debug( "Expected: " + str( onosSet ) )
-                        main.log.debug( "Actual: " + str( current ) )
-                        getResults = main.FALSE
-                else:
-                    # error, set is not a set
-                    main.log.error( "ONOS" + node +
-                                    " has repeat elements in" +
-                                    " set " + onosSetName + ":\n" +
-                                    str( getResponses[ i ] ) )
-                    getResults = main.FALSE
-            elif getResponses[ i ] == main.ERROR:
-                getResults = main.FALSE
-        sizeResponses = []
-        threads = []
-        for i in main.activeNodes:
-            t = main.Thread( target=main.CLIs[i].setTestSize,
-                             name="setTestSize-" + str( i ),
-                             args=[ onosSetName ] )
-            threads.append( t )
-            t.start()
-        for t in threads:
-            t.join()
-            sizeResponses.append( t.result )
-        sizeResults = main.TRUE
-        for i in range( len( main.activeNodes ) ):
-            node = str( main.activeNodes[i] + 1 )
-            if size != sizeResponses[ i ]:
-                sizeResults = main.FALSE
-                main.log.error( "ONOS" + node + " expected a size of " +
-                                str( size ) + " for set " + onosSetName +
-                                " but got " + str( sizeResponses[ i ] ) )
-        retainResults = retainResults and getResults and sizeResults
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=retainResults,
-                                 onpass="Set retain correct",
-                                 onfail="Set retain was incorrect" )
-
-        # Transactional maps
-        main.step( "Partitioned Transactional maps put" )
-        tMapValue = "Testing"
-        numKeys = 100
-        putResult = True
-        node = main.activeNodes[0]
-        putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
-        if putResponses and len( putResponses ) == 100:
-            for i in putResponses:
-                if putResponses[ i ][ 'value' ] != tMapValue:
-                    putResult = False
-        else:
-            putResult = False
-        if not putResult:
-            main.log.debug( "Put response values: " + str( putResponses ) )
-        utilities.assert_equals( expect=True,
-                                 actual=putResult,
-                                 onpass="Partitioned Transactional Map put successful",
-                                 onfail="Partitioned Transactional Map put values are incorrect" )
-
-        main.step( "Partitioned Transactional maps get" )
-        # FIXME: is this sleep needed?
-        time.sleep( 5 )
-
-        getCheck = True
-        for n in range( 1, numKeys + 1 ):
-            getResponses = []
-            threads = []
-            valueCheck = True
-            for i in main.activeNodes:
-                t = main.Thread( target=main.CLIs[i].transactionalMapGet,
-                                 name="TMap-get-" + str( i ),
-                                 args=[ "Key" + str( n ) ] )
-                threads.append( t )
-                t.start()
-            for t in threads:
-                t.join()
-                getResponses.append( t.result )
-            for node in getResponses:
-                if node != tMapValue:
-                    valueCheck = False
-            if not valueCheck:
-                main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
-                main.log.warn( getResponses )
-            getCheck = getCheck and valueCheck
-        utilities.assert_equals( expect=True,
-                                 actual=getCheck,
-                                 onpass="Partitioned Transactional Map get values were correct",
-                                 onfail="Partitioned Transactional Map values incorrect" )
+        main.HA.CASE17( main )