TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py

import os
import imp
import time
import json
import urllib
from core import utilities


class Testcaselib:

    useSSH=False

    @staticmethod
    def initTest( main ):
        """
        - Construct tests variables
        - GIT ( optional )
            - Checkout ONOS master branch
            - Pull latest ONOS code
        - Building ONOS ( optional )
            - Install ONOS package
            - Build ONOS package
        """
        main.step( "Constructing test variables" )
        # Test variables
        main.cellName = main.params[ 'ENV' ][ 'cellName' ]
        main.apps = main.params[ 'ENV' ][ 'cellApps' ]
        main.diff = main.params[ 'ENV' ][ 'diffApps' ]
        gitBranch = main.params[ 'GIT' ][ 'branch' ]
        main.path = os.path.dirname( main.testFile )
        main.dependencyPath = main.path + "/../dependencies/"
        main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
        wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
        main.scale = (main.params[ 'SCALE' ][ 'size' ]).split( "," )
        main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
        # main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
        main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
        main.cellData = { }  # for creating cell file
        main.CLIs = [ ]
        main.ONOSip = [ ]
        main.RESTs = [ ]

        # Assigning ONOS cli handles to a list
        for i in range( 1, main.maxNodes + 1 ):
            main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
            main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) )
            main.ONOSip.append( main.CLIs[ i - 1 ].ip_address )
        # -- INIT SECTION, ONLY RUNS ONCE -- #
        main.startUp = imp.load_source( wrapperFile1,
                                        main.dependencyPath +
                                        wrapperFile1 +
                                        ".py" )

        copyResult1 = main.ONOSbench.scp( main.Mininet1,
                                          main.dependencyPath +
                                          main.topology,
                                          main.Mininet1.home,
                                          direction="to" )
        if main.CLIs:
            stepResult = main.TRUE
        else:
            main.log.error( "Did not properly created list of ONOS CLI handle" )
            stepResult = main.FALSE

        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully construct " +
                                        "test variables ",
                                 onfail="Failed to construct test variables" )

    @staticmethod
    def installOnos( main, vlanCfg=True ):
        """
        - Set up cell
            - Create cell file
            - Set cell file
            - Verify cell file
        - Kill ONOS process
        - Uninstall ONOS cluster
        - Verify ONOS start up
        - Install ONOS cluster
        - Connect to cli
        """
        # main.scale[ 0 ] determines the current number of ONOS controller
        apps = main.apps
        if main.diff:
            apps = main.apps + "," + main.diff
        else:
            main.log.error( "App list is empty" )
        print "NODE COUNT = ", main.numCtrls
        print main.ONOSip
        tempOnosIp = [ ]
        main.dynamicHosts = [ 'in1', 'out1' ]
        for i in range( main.numCtrls ):
            tempOnosIp.append( main.ONOSip[ i ] )
        onosUser = main.params[ 'ENV' ][ 'cellUser' ]
        main.step( "Create and Apply cell file" )
        main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
                                       "temp",
                                       main.Mininet1.ip_address,
                                       apps,
                                       tempOnosIp,
                                       onosUser,
                                       useSSH=Testcaselib.useSSH )
        cellResult = main.ONOSbench.setCell( "temp" )
        verifyResult = main.ONOSbench.verifyCell( )
        stepResult = cellResult and verifyResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully applied cell to " + \
                                        "environment",
                                 onfail="Failed to apply cell to environment " )
        # kill off all onos processes
        main.log.info( "Safety check, killing all ONOS processes" +
                       " before initiating environment setup" )
        for i in range( main.maxNodes ):
            main.ONOSbench.onosDie( main.ONOSip[ i ] )
        main.step( "Create and Install ONOS package" )
        packageResult = main.ONOSbench.buckBuild()

        onosInstallResult = main.TRUE
        for i in range( main.numCtrls ):
            onosInstallResult = onosInstallResult and \
                                main.ONOSbench.onosInstall(
                                        node=main.ONOSip[ i ] )
        stepResult = onosInstallResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully installed ONOS package",
                                 onfail="Failed to install ONOS package" )
        if Testcaselib.useSSH:
            for i in range( main.numCtrls ):
                onosInstallResult = onosInstallResult and \
                                    main.ONOSbench.onosSecureSSH(
                                            node=main.ONOSip[ i ] )
            stepResult = onosInstallResult
            utilities.assert_equals( expect=main.TRUE,
                                     actual=stepResult,
                                     onpass="Successfully secure SSH",
                                     onfail="Failed to secure SSH" )
        main.step( "Starting ONOS service" )
        stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE,
        for i in range( main.numCtrls ):
            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
        if onosIsUp == main.TRUE:
            main.log.report( "ONOS instance is up and ready" )
        else:
            main.log.report( "ONOS instance may not be up, stop and " +
                             "start ONOS again " )
            for i in range( main.numCtrls ):
                stopResult = stopResult and \
                             main.ONOSbench.onosStop( main.ONOSip[ i ] )
            for i in range( main.numCtrls ):
                startResult = startResult and \
                              main.ONOSbench.onosStart( main.ONOSip[ i ] )
        stepResult = onosIsUp and stopResult and startResult

        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="ONOS service is ready",
                                 onfail="ONOS service did not start properly" )
        main.step( "Checking if ONOS CLI is ready" )
        for i in range( main.numCtrls ):
            main.CLIs[ i ].startCellCli( )
            cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
                                                     commandlineTimeout=60,
                                                     onosStartTimeout=100 )
        utilities.assert_equals( expect=main.TRUE,
                                 actual=cliResult,
                                 onpass="ONOS CLI is ready",
                                 onfail="ONOS CLI is not ready" )
        main.active = 0
        for i in range( 10 ):
            ready = True
            output = main.CLIs[ main.active ].summary( )
            if not output:
                ready = False
            if ready:
                break
            time.sleep( 10 )
        utilities.assert_equals( expect=True, actual=ready,
                                 onpass="ONOS summary command succeded",
                                 onfail="ONOS summary command failed" )

        with open( "%s/json/%s.json" % (
                main.dependencyPath, main.cfgName) ) as cfg:
            main.RESTs[ main.active ].setNetCfg( json.load( cfg ) )
        with open( "%s/json/%s.chart" % (
                main.dependencyPath, main.cfgName) ) as chart:
            main.pingChart = json.load( chart )
        if not ready:
            main.log.error( "ONOS startup failed!" )
            main.cleanup( )
            main.exit( )

        for i in range( main.numCtrls ):
            main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.segmentrouting" )
            main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.driver.pipeline" )
            main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.store.group.impl" )
            main.CLIs[ i ].logSet( "DEBUG",
                                   "org.onosproject.net.flowobjective.impl" )

    @staticmethod
    def startMininet( main, topology, args="" ):
        main.step( "Starting Mininet Topology" )
        arg = "--onos %d %s" % (main.numCtrls, args)
        main.topology = topology
        topoResult = main.Mininet1.startNet(
                topoFile=main.Mininet1.home + main.topology, args=arg )
        stepResult = topoResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully loaded topology",
                                 onfail="Failed to load topology" )
        # Exit if topology did not load properly
        if not topoResult:
            main.cleanup( )
            main.exit( )

    @staticmethod
    def checkFlows( main, minFlowCount, dumpflows=True ):
        main.step(
                " Check whether the flow count is bigger than %s" % minFlowCount )
        count = utilities.retry( main.CLIs[ main.active ].checkFlowCount,
                                 main.FALSE,
                                 kwargs={ 'min': minFlowCount },
                                 attempts=10,
                                 sleep=10 )
        utilities.assertEquals( \
                expect=True,
                actual=(count > 0),
                onpass="Flow count looks correct: " + str( count ),
                onfail="Flow count looks wrong: " + str( count ) )

        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.CLIs[ main.active ].checkFlowsState,
                                     main.FALSE,
                                     kwargs={ 'isPENDING': False },
                                     attempts=2,
                                     sleep=10 )
        utilities.assertEquals( \
                expect=main.TRUE,
                actual=flowCheck,
                onpass="Flow status is correct!",
                onfail="Flow status is wrong!" )
        if dumpflows:
            main.ONOSbench.dumpONOSCmd( main.ONOSip[ main.active ],
                                        "flows",
                                        main.logdir,
                                        "flowsBefore" + main.cfgName )
            main.ONOSbench.dumpONOSCmd( main.ONOSip[ main.active ],
                                        "groups",
                                        main.logdir,
                                        "groupsBefore" + main.cfgName )

    @staticmethod
    def pingAll( main, tag="", dumpflows=True ):
        main.log.report( "Check full connectivity" )
        print main.pingChart
        for entry in main.pingChart.itervalues( ):
            print entry
            hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
            expect = main.TRUE if expect else main.FALSE
            main.step( "Connectivity for %s %s" % (str( hosts ), tag) )
            pa = main.Mininet1.pingallHosts( hosts )
            utilities.assert_equals( expect=expect, actual=pa,
                                     onpass="IP connectivity successfully tested",
                                     onfail="IP connectivity failed" )
        if dumpflows:
            main.ONOSbench.dumpONOSCmd( main.ONOSip[ main.active ],
                                        "flows",
                                        main.logdir,
                                        "flowsOn" + tag )
            main.ONOSbench.dumpONOSCmd( main.ONOSip[ main.active ],
                                        "groups",
                                        main.logdir,
                                        "groupsOn" + tag )

    @staticmethod
    def killLink( main, end1, end2, switches, links ):
        """
        end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
        switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
        Kill a link and verify ONOS can see the proper link change
        """
        main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
        main.step( "Kill link between %s and %s" % (end1, end2) )
        LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
        LinkDown = main.Mininet1.link( END2=end1, END1=end2, OPTION="down" )
        main.log.info(
                "Waiting %s seconds for link down to be discovered" % main.linkSleep )
        time.sleep( main.linkSleep )
        topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
                                    main.FALSE,
                                    kwargs={ 'numoswitch': switches,
                                             'numolink': links },
                                    attempts=10,
                                    sleep=main.linkSleep )
        result = topology & LinkDown
        utilities.assert_equals( expect=main.TRUE, actual=result,
                                 onpass="Link down successful",
                                 onfail="Failed to turn off link?" )

    @staticmethod
    def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
                     links ):
        """
        Params:
            end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
            dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
            port1, port2: respective port of the end switches that connects to the link, ex.:'1'
            switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
        Kill a link and verify ONOS can see the proper link change
        """
        main.step( "Restore link between %s and %s" % (end1, end2) )
        result = False
        count = 0
        while True:
            count += 1
            main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
            main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
            main.log.info(
                    "Waiting %s seconds for link up to be discovered" % main.linkSleep )
            time.sleep( main.linkSleep )

            for i in range(0, main.numCtrls):
                onosIsUp = main.ONOSbench.isup( main.ONOSip[ i ] )
                if onosIsUp == main.TRUE:
                    main.CLIs[ i ].portstate( dpid=dpid1, port=port1 )
                    main.CLIs[ i ].portstate( dpid=dpid2, port=port2 )
            time.sleep( main.linkSleep )

            result = main.CLIs[ main.active ].checkStatus( numoswitch=switches,
                                                           numolink=links )
            if count > 5 or result:
                break
        utilities.assert_equals( expect=main.TRUE, actual=result,
                                 onpass="Link up successful",
                                 onfail="Failed to bring link up" )

    @staticmethod
    def killSwitch( main, switch, switches, links ):
        """
        Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
        Completely kill a switch and verify ONOS can see the proper change
        """
        main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
        main.step( "Kill " + switch )
        main.log.info( "Stopping" + switch )
        main.Mininet1.switch( SW=switch, OPTION="stop" )
        # todo make this repeatable
        main.log.info( "Waiting %s seconds for switch down to be discovered" % (
            main.switchSleep) )
        time.sleep( main.switchSleep )
        topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
                                    main.FALSE,
                                    kwargs={ 'numoswitch': switches,
                                             'numolink': links },
                                    attempts=10,
                                    sleep=main.switchSleep )
        utilities.assert_equals( expect=main.TRUE, actual=topology,
                                 onpass="Kill switch successful",
                                 onfail="Failed to kill switch?" )

    @staticmethod
    def recoverSwitch( main, switch, switches, links ):
        """
        Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
        Recover a switch and verify ONOS can see the proper change
        """
        # todo make this repeatable
        main.step( "Recovering " + switch )
        main.log.info( "Starting" + switch )
        main.Mininet1.switch( SW=switch, OPTION="start" )
        main.log.info( "Waiting %s seconds for switch up to be discovered" % (
            main.switchSleep) )
        time.sleep( main.switchSleep )
        topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
                                    main.FALSE,
                                    kwargs={ 'numoswitch': switches,
                                             'numolink': links },
                                    attempts=10,
                                    sleep=main.switchSleep )
        utilities.assert_equals( expect=main.TRUE, actual=topology,
                                 onpass="Switch recovery successful",
                                 onfail="Failed to recover switch?" )

    @staticmethod
    def cleanup( main ):
        """
        Stop Onos-cluster.
        Stops Mininet
        Copies ONOS log
        """
        main.Mininet1.stopNet( )
        main.ONOSbench.scp( main.ONOScli1, "/opt/onos/log/karaf.log",
                            "/tmp/karaf.log", direction="from" )
        main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
                                    copyFileName="karaf.log." + main.cfgName )
        for i in range( main.numCtrls ):
            main.ONOSbench.onosStop( main.ONOSip[ i ] )

    @staticmethod
    def killOnos( main, nodes, switches, links, expNodes ):
        """
        Params: nodes, integer array with position of the ONOS nodes in the CLIs array
        switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
        Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
        """

        main.step( "Killing ONOS instance" )

        for i in nodes:
            killResult = main.ONOSbench.onosDie( main.CLIs[ i ].ip_address )
            utilities.assert_equals( expect=main.TRUE, actual=killResult,
                                     onpass="ONOS instance Killed",
                                     onfail="Error killing ONOS instance" )
            if i == main.active:
                main.active = (i + 1) % main.numCtrls
        time.sleep( 12 )

        if len( nodes ) < main.numCtrls:

            nodesToCheck = []
            for x in range(0, main.numCtrls):
                if x not in nodes:
                    nodesToCheck.append(x)
            nodeResults = utilities.retry( Testcaselib.nodesCheck,
                                           False,
                                           args=[nodesToCheck],
                                           attempts=5,
                                           sleep=10 )
            utilities.assert_equals( expect=True, actual=nodeResults,
                                     onpass="Nodes check successful",
                                     onfail="Nodes check NOT successful" )

            if not nodeResults:
                for i in nodes:
                    cli = main.CLIs[i]
                    main.log.debug( "{} components not ACTIVE: \n{}".format(
                        cli.name,
                        cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
                main.log.error( "Failed to kill ONOS, stopping test" )
                main.cleanup()
                main.exit()

            topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
                                        main.FALSE,
                                        kwargs={ 'numoswitch': switches,
                                                 'numolink': links,
                                                 'numoctrl': expNodes },
                                        attempts=10,
                                        sleep=12 )
            utilities.assert_equals( expect=main.TRUE, actual=topology,
                                     onpass="ONOS Instance down successful",
                                     onfail="Failed to turn off ONOS Instance" )
        else:
            main.active = -1

    @staticmethod
    def recoverOnos( main, nodes, switches, links, expNodes ):
        """
        Params: nodes, integer array with position of the ONOS nodes in the CLIs array
        switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
        Recover an ONOS instance and verify the ONOS cluster can see the proper change
        """
        main.step( "Recovering ONOS instance" )
        [ main.ONOSbench.onosStart( main.CLIs[ i ].ip_address ) for i in nodes ]
        for i in nodes:
            isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
            utilities.assert_equals( expect=main.TRUE, actual=isUp,
                                     onpass="ONOS service is ready",
                                     onfail="ONOS service did not start properly" )
        for i in nodes:
            main.step( "Checking if ONOS CLI is ready" )
            main.CLIs[ i ].startCellCli( )
            cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
                                                     commandlineTimeout=60,
                                                     onosStartTimeout=100 )
            utilities.assert_equals( expect=main.TRUE,
                                     actual=cliResult,
                                     onpass="ONOS CLI is ready",
                                     onfail="ONOS CLI is not ready" )
            main.active = i if main.active == -1 else main.active

        main.step( "Checking ONOS nodes" )
        nodeResults = utilities.retry( Testcaselib.nodesCheck,
                                       False,
                                       args=[nodes],
                                       attempts=5,
                                       sleep=10 )
        utilities.assert_equals( expect=True, actual=nodeResults,
                                 onpass="Nodes check successful",
                                 onfail="Nodes check NOT successful" )

        if not nodeResults:
            for i in nodes:
                cli = main.CLIs[i]
                main.log.debug( "{} components not ACTIVE: \n{}".format(
                    cli.name,
                    cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
            main.log.error( "Failed to start ONOS, stopping test" )
            main.cleanup()
            main.exit()

        topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
                                    main.FALSE,
                                    kwargs={ 'numoswitch': switches,
                                             'numolink': links,
                                             'numoctrl': expNodes },
                                    attempts=10,
                                    sleep=12 )
        utilities.assert_equals( expect=main.TRUE, actual=topology,
                                 onpass="ONOS Instance down successful",
                                 onfail="Failed to turn off ONOS Instance" )
        for i in range( 10 ):
            ready = True
            output = main.CLIs[ main.active ].summary( )
            if not output:
                ready = False
            if ready:
                break
            time.sleep( 10 )
        utilities.assert_equals( expect=True, actual=ready,
                                 onpass="ONOS summary command succeded",
                                 onfail="ONOS summary command failed" )
        if not ready:
            main.log.error( "ONOS startup failed!" )
            main.cleanup( )
            main.exit( )

    @staticmethod
    def addHostCfg( main ):
        """
        Adds Host Configuration to ONOS
        Updates expected state of the network (pingChart)
        """
        import json
        hostCfg = { }
        with open( main.dependencyPath + "/json/extra.json" ) as template:
            hostCfg = json.load( template )
        main.pingChart[ 'ip' ][ 'hosts' ] += [ 'in1' ]
        main.step( "Pushing new configuration" )
        mac, cfg = hostCfg[ 'hosts' ].popitem( )
        main.RESTs[ main.active ].setNetCfg( cfg[ 'basic' ],
                                             subjectClass="hosts",
                                             subjectKey=urllib.quote( mac,
                                                                      safe='' ),
                                             configKey="basic" )
        main.pingChart[ 'ip' ][ 'hosts' ] += [ 'out1' ]
        main.step( "Pushing new configuration" )
        mac, cfg = hostCfg[ 'hosts' ].popitem( )
        main.RESTs[ main.active ].setNetCfg( cfg[ 'basic' ],
                                             subjectClass="hosts",
                                             subjectKey=urllib.quote( mac,
                                                                      safe='' ),
                                             configKey="basic" )
        main.pingChart.update( { 'vlan1': { "expect": "True",
                                            "hosts": [ "olt1", "vsg1" ] } } )
        main.pingChart[ 'vlan5' ][ 'expect' ] = 0
        main.pingChart[ 'vlan10' ][ 'expect' ] = 0
        ports = "[%s,%s]" % (5, 6)
        cfg = '{"of:0000000000000001":[{"vlan":1,"ports":%s,"name":"OLT 1"}]}' % ports
        main.RESTs[ main.active ].setNetCfg( json.loads( cfg ),
                                             subjectClass="apps",
                                             subjectKey="org.onosproject.segmentrouting",
                                             configKey="xconnect" )

    @staticmethod
    def delHostCfg( main ):
        """
        Removest Host Configuration from ONOS
        Updates expected state of the network (pingChart)
        """
        import json
        hostCfg = { }
        with open( main.dependencyPath + "/json/extra.json" ) as template:
            hostCfg = json.load( template )
        main.step( "Removing host configuration" )
        main.pingChart[ 'ip' ][ 'expect' ] = 0
        mac, cfg = hostCfg[ 'hosts' ].popitem( )
        main.RESTs[ main.active ].removeNetCfg( subjectClass="hosts",
                                                subjectKey=urllib.quote(
                                                        mac,
                                                        safe='' ),
                                                configKey="basic" )
        main.step( "Removing configuration" )
        main.pingChart[ 'ip' ][ 'expect' ] = 0
        mac, cfg = hostCfg[ 'hosts' ].popitem( )
        main.RESTs[ main.active ].removeNetCfg( subjectClass="hosts",
                                                subjectKey=urllib.quote(
                                                        mac,
                                                        safe='' ),
                                                configKey="basic" )
        main.step( "Removing vlan configuration" )
        main.pingChart[ 'vlan1' ][ 'expect' ] = 0
        main.RESTs[ main.active ].removeNetCfg( subjectClass="apps",
                                                subjectKey="org.onosproject.segmentrouting",
                                                configKey="xconnect" )
    @staticmethod
    def nodesCheck( nodes ):
        nodesOutput = []
        results = True
        threads = []
        for i in nodes:
            t = main.Thread( target=main.CLIs[i].nodes,
                             name="nodes-" + str( i ),
                             args=[ ] )
            threads.append( t )
            t.start()

        for t in threads:
            t.join()
            nodesOutput.append( t.result )
        ips = [ main.ONOSip[ node ] for node in nodes ]
        ips.sort()
        for i in nodesOutput:
            try:
                current = json.loads( i )
                activeIps = []
                currentResult = False
                for node in current:
                    if node['state'] == 'READY':
                        activeIps.append( node['ip'] )
                currentResult = True
                for ip in ips:
                    if ip not in activeIps:
                        currentResult = False
                        break
            except ( ValueError, TypeError ):
                main.log.error( "Error parsing nodes output" )
                main.log.warn( repr( i ) )
                currentResult = False
            results = results and currentResult
        return results