TestON/tests/SCPF/SCPFmastershipFailoverLat/SCPFmastershipFailoverLat.py - OnosSystemTest - Gitiles

 """
 Copyright 2017 Open Networking Foundation ( ONF )

 Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
 the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
 or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>

     TestON is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 2 of the License, or
     ( at your option ) any later version.

     TestON is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.

     You should have received a copy of the GNU General Public License
     along with TestON.  If not, see <http://www.gnu.org/licenses/>.
 """
 class SCPFmastershipFailoverLat:

     def __init__( self ):
         self.default = ''

     def CASE0( self, main ):
         import os
         import imp
         """
         - GIT
         - BUILDING ONOS
             Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
             This step is usually skipped. Because in a Jenkins driven automated
             test env. We want Jenkins jobs to pull&build for flexibility to handle
             different versions of ONOS.
         - Construct tests variables
         """
         try:
             from tests.dependencies.ONOSSetup import ONOSSetup
             main.testSetUp = ONOSSetup()
         except ImportError:
             main.log.error( "ONOSSetup not found. exiting the test" )
             main.exit()
         main.testSetUp.envSetupDescription()
         stepResult = main.FALSE
         try:
             main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
             main.cellName = main.params[ 'ENV' ][ 'cellName' ]
             main.apps = main.params[ 'ENV' ][ 'cellApps' ]
             main.scale = ( main.params[ 'SCALE' ] ).split( "," )
             main.ofpRoleRequest = main.params[ 'TSHARK' ][ 'ofpRoleRequest' ]
             main.tsharkResultPath = main.params[ 'TSHARK' ][ 'tsharkResultPath' ]
             main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
             main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
             main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
             main.maxScale = int( main.params[ 'max' ] )
             main.timeout = int( main.params[ 'TIMEOUT' ][ 'timeout' ] )
             main.MNSleep = int( main.params[ 'SLEEP' ][ 'mininet' ] )
             main.recoverySleep = int( main.params[ 'SLEEP' ][ 'recovery' ] )
             main.debug = main.params[ 'TEST' ][ 'debug' ]
             main.failoverSleep = int( main.params[ 'SLEEP' ][ 'failover' ] )
             main.switchID = main.params[ 'SWITCH' ][ 'id' ]
             main.topologySwitchCount = main.params[ 'TOPOLOGY' ][ 'switchCount' ]
             main.topologyType = main.params[ 'TOPOLOGY' ][ 'type' ]
             main.nodeNumToKill = int( main.params[ 'KILL' ][ 'nodeNum' ] )
             main.failPercent = float( main.params[ 'TEST' ][ 'failPercent' ] )

             if main.debug == "True":
                 main.debug = True
             else:
                 main.debug = False

             stepResult = main.testSetUp.envSetup()
             main.log.info( "Create Database file " + main.dbFileName )
             resultsDB = open( main.dbFileName, "w+" )
             resultsDB.close()

         except Exception as e:
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )

     def CASE1( self, main ):
         # Clean up test environment and set up
         import time
         main.testSetUp.ONOSSetUp( main.Cluster, True,
                                   cellName=main.cellName, killRemoveMax=False )
         try:
             from tests.dependencies.utils import Utils
         except ImportError:
             main.log.error( "Utils not found exiting the test" )
             main.exit()
         try:
             main.Utils
         except ( NameError, AttributeError ):
             main.Utils = Utils()
         main.Utils.mininetCleanup( main.Mininet1 )

         main.step( "Starting up Mininet from command." )

         mnCmd = " mn " + " --topo " + main.topologyType + "," + main.topologySwitchCount
         for ctrl in main.Cluster.active():
             mnCmd += " --controller remote,ip=" + ctrl.ipAddress

         stepResult = main.Mininet1.startNet( mnCmd=mnCmd )

         utilities.assert_equals( expect=main.TRUE,
                                  actual=stepResult,
                                  onpass="Mininet was set up correctly.",
                                  onfail="Mininet was NOT set up correctly." )

     def CASE2( self, main ):
         """
         Kill ONOS node, and measure the latency for INSTANCE_DEACTIVATED, MASTER_CHANGED, and role request
         ( tshark time ), then bring the node back up.
         """
         import time
         import datetime
         import numpy
         from tests.HA.dependencies.HA import HA

         main.HA = HA()

         main.latencyData = { 'kill_to_deactivation': [],
                              'deactivation_to_role_request': [] }

         main.failCounter = 0
         passingResult = True
         criticalError = False

         main.step( "Gathering data starting with "
                    + str( main.warmUp )
                    + " warm ups and a sample size of "
                    + str( main.sampleSize ) )

         for iteration in range( 0, main.sampleSize + main.warmUp ):

             main.log.info( "==========================================" )
             main.log.info( "================iteration:{}==============".format( str( iteration + 1 ) ) )

             ip_address = main.Cluster.active( 0 ).ipAddress
             strNodeNumToKill = str( main.nodeNumToKill )

             main.log.info( "Assigning mastership to ONOS node " + strNodeNumToKill )
             main.Cluster.active( 0 ).CLI.deviceRole( main.switchID, ip_address )

             main.log.info( "Sleeping for " + str( main.recoverySleep ) + " seconds..." )
             time.sleep( main.recoverySleep )
             mastershipCheck = main.Cluster.active( 0 ).CLI.getMaster( main.switchID ) == ip_address

             if not mastershipCheck:
                 main.log.warn( "Mastership is NOT as expected." )

             with open( main.tsharkResultPath, "w" ) as tshark:
                 tshark.write( "" )
             main.log.info( "Starting tshark capture." )
             main.ONOSbench.tsharkGrep( main.ofpRoleRequest, main.tsharkResultPath )
             time1 = time.time() * 1000.0

             # Kill an ONOS node
             main.log.info( "Killing ONOS node " + strNodeNumToKill + "." )
             killresult = main.ONOSbench.onosKill( ip_address )
             main.Cluster.runningNodes[ main.nodeNumToKill ].active = False

             # Stop an ONOS node
             main.log.info( "Stopping ONOS node " + strNodeNumToKill + "." )
             stopresult = main.ONOSbench.onosStop( ip_address )

             killStopResult = stopresult == killresult and True

             if not killStopResult:
                 main.log.error( "ONOS node was NOT successfully stopped and killed." )
                 criticalError = True

             time.sleep( main.failoverSleep )

             # Stop tshark and get times
             main.log.info( "Stopping tshark." )
             main.ONOSbench.tsharkStop()

             masterChangedLats = []
             instanceDeactivatedLats = []

             main.log.info( "Obtaining latencies from 'events' output." )
             for CLInum in range( 0, main.Cluster.numCtrls - 1 ):
                 eventOutput = main.Cluster.active( CLInum ).CLI.events( args='-a' ).split( "\r\n" )
                 for line in reversed( eventOutput ):
                     timestamp = line[ :23 ] if line[ 19 ] != '-' else line[ :19 ] + '.000'
                     timestamp = float( datetime.datetime.strptime( timestamp, "%Y-%m-%dT%H:%M:%S.%f" ).strftime( '%s.%f' ) ) * 1000.0
                     if timestamp - time1 >= 0:
                         if "INSTANCE_DEACTIVATED" in line:
                             instanceDeactivatedLats.append( timestamp - time1 )
                         elif "MASTER_CHANGED" in line:
                             masterChangedLats.append( timestamp - time1 )
                     else:
                         break

             if instanceDeactivatedLats and masterChangedLats:
                 instanceDeactivatedLats.sort()
                 masterChangedLats.sort()
                 instanceDeactivated = instanceDeactivatedLats[ 0 ]
                 masterChanged = masterChangedLats[ 0 ]
                 eventLatCheck = True
             else:
                 eventLatCheck = False
                 main.log.warn( "Latencies were NOT obtained from 'events' successfully." )

             main.log.info( "Obtain latency from tshark output." )
             tsharkLatCheck = True
             with open( main.tsharkResultPath, "r" ) as resultFile:
                 resultText = resultFile.readline()
                 main.log.info( "Capture result: " + resultText )
                 resultText = resultText.split()
                 if len( resultText ) > 1:
                     roleRequestLat = int( float( resultText[ 1 ] ) * 1000.0 ) - time1
                     resultFile.close()
                 else:
                     main.log.error( "Tshark output file is NOT as expected." )
                     tsharkLatCheck = False
             if not tsharkLatCheck:
                 main.log.warn( "Latency was NOT obtained from tshark successfully." )

             validDataCheck = False
             if tsharkLatCheck:
                 main.log.info( "instanceDeactivated: " + str( instanceDeactivated ) )
                 main.log.info( "masterChanged: " + str( masterChanged ) )
                 main.log.info( "roleRequestLat: " + str( roleRequestLat ) )
                 if iteration >= main.warmUp:
                     main.log.info( "Verifying that the data are valid." )  # Don't record data during a warm-up
                     validDataCheck = roleRequestLat >= 0 and \
                                      instanceDeactivated >= 0 and \
                                      masterChanged >= 0
                     if not validDataCheck:
                         main.log.warn( "Data are NOT valid." )

                     if eventLatCheck and tsharkLatCheck and validDataCheck:
                         main.log.info( "Saving data..." )
                         if roleRequestLat >= instanceDeactivated:
                             main.latencyData[ 'kill_to_deactivation' ].append( instanceDeactivated )
                             main.latencyData[ 'deactivation_to_role_request' ].append( roleRequestLat - instanceDeactivated )
                         else:
                             main.latencyData[ 'kill_to_deactivation' ].append( roleRequestLat )
                             main.latencyData[ 'deactivation_to_role_request' ].append( 0 )
                         main.log.info( "kill_to_deactivation: " + str( main.latencyData[ 'kill_to_deactivation' ][ -1 ] ) )
                         main.log.info( "deactivation_to_role_request: " + str( main.latencyData[ 'deactivation_to_role_request' ][ -1 ] ) )

             # Restart ONOS node
             main.log.info( "Restart ONOS node " + strNodeNumToKill + " and checking status of restart." )
             startResult = main.ONOSbench.onosStart( ip_address )

             if not startResult:
                 main.log.error( "ONOS nodes NOT successfully started." )
                 criticalError = True

             # Check if ONOS is up yet
             main.log.info( "Checking if ONOS node " + strNodeNumToKill + " is up." )
             upResult = main.ONOSbench.isup( ip_address )

             if not upResult:
                 main.log.error( "ONOS did NOT successfully restart." )
                 criticalError = True

             # Restart CLI
             main.log.info( "Restarting ONOS node " + strNodeNumToKill + "'s main.CLI." )
             cliResult = main.Cluster.active( main.nodeNumToKill ).CLI.startOnosCli( ip_address )
             main.Cluster.runningNodes[ main.nodeNumToKill ] .active = True

             if not cliResult:
                 main.log.error( "ONOS CLI did NOT successfully restart." )
                 criticalError = True

             main.log.info( "Checking ONOS nodes." )
             nodeResults = utilities.retry( main.Cluster.nodesCheck,
                                            False,
                                            sleep=1,
                                            attempts=3 )

             if not nodeResults:
                 main.log.error( "Nodes check NOT successful." )
                 criticalError = True

             main.log.info( "Sleeping for " + str( main.recoverySleep ) + " seconds..." )
             time.sleep( main.recoverySleep )

             if not ( mastershipCheck and
                      eventLatCheck and
                      tsharkLatCheck and
                      validDataCheck ) and \
                      iteration >= main.warmUp:
                 main.failCounter += 1
                 main.log.warn( "Iteration failed. Failure count: " + str( main.failCounter ) )
             if float( main.failCounter ) / float( main.sampleSize ) >= main.failPercent or criticalError:
                 main.log.error( str( main.failPercent * 100 )
                                 + "% or more of data is invalid, or a critical error has occurred." )
                 passingResult = False
                 break

         utilities.assert_equals( expect=True, actual=passingResult,
                                  onpass="Node scaling "
                                         + str( main.Cluster.numCtrls )
                                         + " data gathering was successful.",
                                  onfail="Node scaling "
                                         + str( main.Cluster.numCtrls )
                                         + " data gathering FAILED. Stopping test." )
         if not passingResult:
             main.cleanAndExit()

     def CASE3( self, main ):
         """
         Write results to database file.
         Omit this case if you don't want to write to database.
         """
         import numpy
         result = { 'avg': {}, 'stddev': {} }

         for i in main.latencyData:
             result[ 'avg' ][ i ] = numpy.average( main.latencyData[ i ] )
             result[ 'stddev' ][ i ] = numpy.std( main.latencyData[ i ] )

         main.log.info( "result: " + str( result ) )
         with open( main.dbFileName, "a" ) as dbFile:
             strToWrite = str( main.Cluster.numCtrls ) + ",'baremetal1'"
             strToWrite += ",'" + main.commit.split()[1] + "'"
             for i in result:
                 for j in result[ i ]:
                     strToWrite += "," + str( result[ i ][ j ] )
             strToWrite += "\n"
             dbFile.write( strToWrite )
             dbFile.close()
	"""
	Copyright 2017 Open Networking Foundation ( ONF )

	Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
	the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
	or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>

	TestON is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 2 of the License, or
	( at your option ) any later version.

	TestON is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with TestON. If not, see <http://www.gnu.org/licenses/>.
	"""
	class SCPFmastershipFailoverLat:

	def __init__( self ):
	self.default = ''

	def CASE0( self, main ):
	import os
	import imp
	"""
	- GIT
	- BUILDING ONOS
	Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
	This step is usually skipped. Because in a Jenkins driven automated
	test env. We want Jenkins jobs to pull&build for flexibility to handle
	different versions of ONOS.
	- Construct tests variables
	"""
	try:
	from tests.dependencies.ONOSSetup import ONOSSetup
	main.testSetUp = ONOSSetup()
	except ImportError:
	main.log.error( "ONOSSetup not found. exiting the test" )
	main.exit()
	main.testSetUp.envSetupDescription()
	stepResult = main.FALSE
	try:
	main.MN1Ip = main.params[ 'MN' ][ 'ip1' ]
	main.cellName = main.params[ 'ENV' ][ 'cellName' ]
	main.apps = main.params[ 'ENV' ][ 'cellApps' ]
	main.scale = ( main.params[ 'SCALE' ] ).split( "," )
	main.ofpRoleRequest = main.params[ 'TSHARK' ][ 'ofpRoleRequest' ]
	main.tsharkResultPath = main.params[ 'TSHARK' ][ 'tsharkResultPath' ]
	main.sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
	main.warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
	main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
	main.maxScale = int( main.params[ 'max' ] )
	main.timeout = int( main.params[ 'TIMEOUT' ][ 'timeout' ] )
	main.MNSleep = int( main.params[ 'SLEEP' ][ 'mininet' ] )
	main.recoverySleep = int( main.params[ 'SLEEP' ][ 'recovery' ] )
	main.debug = main.params[ 'TEST' ][ 'debug' ]
	main.failoverSleep = int( main.params[ 'SLEEP' ][ 'failover' ] )
	main.switchID = main.params[ 'SWITCH' ][ 'id' ]
	main.topologySwitchCount = main.params[ 'TOPOLOGY' ][ 'switchCount' ]
	main.topologyType = main.params[ 'TOPOLOGY' ][ 'type' ]
	main.nodeNumToKill = int( main.params[ 'KILL' ][ 'nodeNum' ] )
	main.failPercent = float( main.params[ 'TEST' ][ 'failPercent' ] )

	if main.debug == "True":
	main.debug = True
	else:
	main.debug = False

	stepResult = main.testSetUp.envSetup()
	main.log.info( "Create Database file " + main.dbFileName )
	resultsDB = open( main.dbFileName, "w+" )
	resultsDB.close()

	except Exception as e:
	main.testSetUp.envSetupException( e )
	main.testSetUp.evnSetupConclusion( stepResult )

	def CASE1( self, main ):
	# Clean up test environment and set up
	import time
	main.testSetUp.ONOSSetUp( main.Cluster, True,
	cellName=main.cellName, killRemoveMax=False )
	try:
	from tests.dependencies.utils import Utils
	except ImportError:
	main.log.error( "Utils not found exiting the test" )
	main.exit()
	try:
	main.Utils
	except ( NameError, AttributeError ):
	main.Utils = Utils()
	main.Utils.mininetCleanup( main.Mininet1 )

	main.step( "Starting up Mininet from command." )

	mnCmd = " mn " + " --topo " + main.topologyType + "," + main.topologySwitchCount
	for ctrl in main.Cluster.active():
	mnCmd += " --controller remote,ip=" + ctrl.ipAddress

	stepResult = main.Mininet1.startNet( mnCmd=mnCmd )

	utilities.assert_equals( expect=main.TRUE,
	actual=stepResult,
	onpass="Mininet was set up correctly.",
	onfail="Mininet was NOT set up correctly." )

	def CASE2( self, main ):
	"""
	Kill ONOS node, and measure the latency for INSTANCE_DEACTIVATED, MASTER_CHANGED, and role request
	( tshark time ), then bring the node back up.
	"""
	import time
	import datetime
	import numpy
	from tests.HA.dependencies.HA import HA

	main.HA = HA()

	main.latencyData = { 'kill_to_deactivation': [],
	'deactivation_to_role_request': [] }

	main.failCounter = 0
	passingResult = True
	criticalError = False

	main.step( "Gathering data starting with "
	+ str( main.warmUp )
	+ " warm ups and a sample size of "
	+ str( main.sampleSize ) )

	for iteration in range( 0, main.sampleSize + main.warmUp ):

	main.log.info( "==========================================" )
	main.log.info( "================iteration:{}==============".format( str( iteration + 1 ) ) )

	ip_address = main.Cluster.active( 0 ).ipAddress
	strNodeNumToKill = str( main.nodeNumToKill )

	main.log.info( "Assigning mastership to ONOS node " + strNodeNumToKill )
	main.Cluster.active( 0 ).CLI.deviceRole( main.switchID, ip_address )

	main.log.info( "Sleeping for " + str( main.recoverySleep ) + " seconds..." )
	time.sleep( main.recoverySleep )
	mastershipCheck = main.Cluster.active( 0 ).CLI.getMaster( main.switchID ) == ip_address

	if not mastershipCheck:
	main.log.warn( "Mastership is NOT as expected." )

	with open( main.tsharkResultPath, "w" ) as tshark:
	tshark.write( "" )
	main.log.info( "Starting tshark capture." )
	main.ONOSbench.tsharkGrep( main.ofpRoleRequest, main.tsharkResultPath )
	time1 = time.time() * 1000.0

	# Kill an ONOS node
	main.log.info( "Killing ONOS node " + strNodeNumToKill + "." )
	killresult = main.ONOSbench.onosKill( ip_address )
	main.Cluster.runningNodes[ main.nodeNumToKill ].active = False

	# Stop an ONOS node
	main.log.info( "Stopping ONOS node " + strNodeNumToKill + "." )
	stopresult = main.ONOSbench.onosStop( ip_address )

	killStopResult = stopresult == killresult and True

	if not killStopResult:
	main.log.error( "ONOS node was NOT successfully stopped and killed." )
	criticalError = True

	time.sleep( main.failoverSleep )

	# Stop tshark and get times
	main.log.info( "Stopping tshark." )
	main.ONOSbench.tsharkStop()

	masterChangedLats = []
	instanceDeactivatedLats = []

	main.log.info( "Obtaining latencies from 'events' output." )
	for CLInum in range( 0, main.Cluster.numCtrls - 1 ):
	eventOutput = main.Cluster.active( CLInum ).CLI.events( args='-a' ).split( "\r\n" )
	for line in reversed( eventOutput ):
	timestamp = line[ :23 ] if line[ 19 ] != '-' else line[ :19 ] + '.000'
	timestamp = float( datetime.datetime.strptime( timestamp, "%Y-%m-%dT%H:%M:%S.%f" ).strftime( '%s.%f' ) ) * 1000.0
	if timestamp - time1 >= 0:
	if "INSTANCE_DEACTIVATED" in line:
	instanceDeactivatedLats.append( timestamp - time1 )
	elif "MASTER_CHANGED" in line:
	masterChangedLats.append( timestamp - time1 )
	else:
	break

	if instanceDeactivatedLats and masterChangedLats:
	instanceDeactivatedLats.sort()
	masterChangedLats.sort()
	instanceDeactivated = instanceDeactivatedLats[ 0 ]
	masterChanged = masterChangedLats[ 0 ]
	eventLatCheck = True
	else:
	eventLatCheck = False
	main.log.warn( "Latencies were NOT obtained from 'events' successfully." )

	main.log.info( "Obtain latency from tshark output." )
	tsharkLatCheck = True
	with open( main.tsharkResultPath, "r" ) as resultFile:
	resultText = resultFile.readline()
	main.log.info( "Capture result: " + resultText )
	resultText = resultText.split()
	if len( resultText ) > 1:
	roleRequestLat = int( float( resultText[ 1 ] ) * 1000.0 ) - time1
	resultFile.close()
	else:
	main.log.error( "Tshark output file is NOT as expected." )
	tsharkLatCheck = False
	if not tsharkLatCheck:
	main.log.warn( "Latency was NOT obtained from tshark successfully." )

	validDataCheck = False
	if tsharkLatCheck:
	main.log.info( "instanceDeactivated: " + str( instanceDeactivated ) )
	main.log.info( "masterChanged: " + str( masterChanged ) )
	main.log.info( "roleRequestLat: " + str( roleRequestLat ) )
	if iteration >= main.warmUp:
	main.log.info( "Verifying that the data are valid." ) # Don't record data during a warm-up
	validDataCheck = roleRequestLat >= 0 and \
	instanceDeactivated >= 0 and \
	masterChanged >= 0
	if not validDataCheck:
	main.log.warn( "Data are NOT valid." )

	if eventLatCheck and tsharkLatCheck and validDataCheck:
	main.log.info( "Saving data..." )
	if roleRequestLat >= instanceDeactivated:
	main.latencyData[ 'kill_to_deactivation' ].append( instanceDeactivated )
	main.latencyData[ 'deactivation_to_role_request' ].append( roleRequestLat - instanceDeactivated )
	else:
	main.latencyData[ 'kill_to_deactivation' ].append( roleRequestLat )
	main.latencyData[ 'deactivation_to_role_request' ].append( 0 )
	main.log.info( "kill_to_deactivation: " + str( main.latencyData[ 'kill_to_deactivation' ][ -1 ] ) )
	main.log.info( "deactivation_to_role_request: " + str( main.latencyData[ 'deactivation_to_role_request' ][ -1 ] ) )

	# Restart ONOS node
	main.log.info( "Restart ONOS node " + strNodeNumToKill + " and checking status of restart." )
	startResult = main.ONOSbench.onosStart( ip_address )

	if not startResult:
	main.log.error( "ONOS nodes NOT successfully started." )
	criticalError = True

	# Check if ONOS is up yet
	main.log.info( "Checking if ONOS node " + strNodeNumToKill + " is up." )
	upResult = main.ONOSbench.isup( ip_address )

	if not upResult:
	main.log.error( "ONOS did NOT successfully restart." )
	criticalError = True

	# Restart CLI
	main.log.info( "Restarting ONOS node " + strNodeNumToKill + "'s main.CLI." )
	cliResult = main.Cluster.active( main.nodeNumToKill ).CLI.startOnosCli( ip_address )
	main.Cluster.runningNodes[ main.nodeNumToKill ] .active = True

	if not cliResult:
	main.log.error( "ONOS CLI did NOT successfully restart." )
	criticalError = True

	main.log.info( "Checking ONOS nodes." )
	nodeResults = utilities.retry( main.Cluster.nodesCheck,
	False,
	sleep=1,
	attempts=3 )

	if not nodeResults:
	main.log.error( "Nodes check NOT successful." )
	criticalError = True

	main.log.info( "Sleeping for " + str( main.recoverySleep ) + " seconds..." )
	time.sleep( main.recoverySleep )

	if not ( mastershipCheck and
	eventLatCheck and
	tsharkLatCheck and
	validDataCheck ) and \
	iteration >= main.warmUp:
	main.failCounter += 1
	main.log.warn( "Iteration failed. Failure count: " + str( main.failCounter ) )
	if float( main.failCounter ) / float( main.sampleSize ) >= main.failPercent or criticalError:
	main.log.error( str( main.failPercent * 100 )
	+ "% or more of data is invalid, or a critical error has occurred." )
	passingResult = False
	break

	utilities.assert_equals( expect=True, actual=passingResult,
	onpass="Node scaling "
	+ str( main.Cluster.numCtrls )
	+ " data gathering was successful.",
	onfail="Node scaling "
	+ str( main.Cluster.numCtrls )
	+ " data gathering FAILED. Stopping test." )
	if not passingResult:
	main.cleanAndExit()

	def CASE3( self, main ):
	"""
	Write results to database file.
	Omit this case if you don't want to write to database.
	"""
	import numpy
	result = { 'avg': {}, 'stddev': {} }

	for i in main.latencyData:
	result[ 'avg' ][ i ] = numpy.average( main.latencyData[ i ] )
	result[ 'stddev' ][ i ] = numpy.std( main.latencyData[ i ] )

	main.log.info( "result: " + str( result ) )
	with open( main.dbFileName, "a" ) as dbFile:
	strToWrite = str( main.Cluster.numCtrls ) + ",'baremetal1'"
	strToWrite += ",'" + main.commit.split()[1] + "'"
	for i in result:
	for j in result[ i ]:
	strToWrite += "," + str( result[ i ][ j ] )
	strToWrite += "\n"
	dbFile.write( strToWrite )
	dbFile.close()