Initial implementation of ONOS cluster driver
- Create CLI, REST, and "Bench" components for a cluster
- Return driver object when it is created
- Add __str__ and __repr__ implementations for drivers
- Add first pass at a cluster class
- Prototype with clustered Sample test
- Prototype with HAsanity test
- Add new Exception class for SkipCase
Change-Id: I32ee7cf655ab9a2a5cfccf5f891ca71a6a70c1ee
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.py b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
index e1a66a3..c17e11d 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
@@ -67,13 +67,8 @@
start cli sessions
start tcpdump
"""
- import imp
- import pexpect
- import time
- import json
main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
"initialization" )
- # set global variables
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
@@ -85,25 +80,21 @@
main.exit()
main.testSetUp.envSetupDescription()
try:
+ from dependencies.Cluster import Cluster
from tests.HA.dependencies.HA import HA
main.HA = HA()
- # load some variables from the params file
+ main.Cluster = Cluster( main.ONOScell.nodes )
cellName = main.params[ 'ENV' ][ 'cellName' ]
main.apps = main.params[ 'ENV' ][ 'appString' ]
- main.numCtrls = int( main.params[ 'num_controllers' ] )
- if main.ONOSbench.maxNodes and\
- main.ONOSbench.maxNodes < main.numCtrls:
- main.numCtrls = int( main.ONOSbench.maxNodes )
- main.maxNodes = main.numCtrls
- stepResult = main.testSetUp.envSetup( hasNode=True )
+ stepResult = main.testSetUp.envSetup( main.Cluster, hasNode=True )
except Exception as e:
main.testSetUp.envSetupException( e )
main.testSetUp.evnSetupConclusion( stepResult )
main.HA.generateGraph( "HAstopNodes" )
- main.testSetUp.ONOSSetUp( main.Mininet1, cellName=cellName, removeLog=True,
- extraApply=main.HA.customizeOnosGenPartitions,
- extraClean=main.HA.cleanUpGenPartition )
+ main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
+ extraApply=main.HA.customizeOnosGenPartitions,
+ extraClean=main.HA.cleanUpGenPartition )
main.HA.initialSetUp()
@@ -119,20 +110,17 @@
"""
main.HA.assignMastership( main )
-
def CASE3( self, main ):
"""
Assign intents
"""
main.HA.assignIntents( main )
-
-
def CASE4( self, main ):
"""
Ping across added host intents
"""
- main.HA.pingAcrossHostIntent( main, True, False )
+ main.HA.pingAcrossHostIntent( main )
def CASE5( self, main ):
"""
@@ -147,28 +135,26 @@
assert main.numCtrls, "main.numCtrls not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- assert main.CLIs, "main.CLIs not defined"
- assert main.nodes, "main.nodes not defined"
main.case( "Stop minority of ONOS nodes" )
main.step( "Checking ONOS Logs for errors" )
- for node in main.nodes:
- main.log.debug( "Checking logs for errors on " + node.name + ":" )
- main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
+ for ctrl in main.Cluster.active():
+ main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
+ main.log.warn( ctrl.checkLogs( ctrl.ipAddress ) )
- n = len( main.nodes ) # Number of nodes
+ n = len( main.Cluster.controllers ) # Number of nodes
p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
- main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
+ main.kill = [ main.Cluster.controllers[ 0 ] ] # ONOS node to kill, listed by index in main.nodes
if n > 3:
- main.kill.append( p - 1 )
+ main.kill.append( main.Cluster.controllers[ p - 1 ] )
# NOTE: This only works for cluster sizes of 3,5, or 7.
- main.step( "Stopping " + str( len( main.kill ) ) + " ONOS nodes" )
+ main.step( "Stopping nodes: " + str( main.kill ) )
killResults = main.TRUE
- for i in main.kill:
+ for ctrl in main.kill:
killResults = killResults and\
- main.ONOSbench.onosStop( main.nodes[ i ].ip_address )
- main.activeNodes.remove( i )
+ ctrl.onosStop( ctrl.ipAddress )
+ ctrl.active = False
utilities.assert_equals( expect=main.TRUE, actual=killResults,
onpass="ONOS nodes stopped successfully",
onfail="ONOS nodes NOT successfully stopped" )
@@ -176,7 +162,7 @@
main.step( "Checking ONOS nodes" )
nodeResults = utilities.retry( main.HA.nodesCheck,
False,
- args=[ main.activeNodes ],
+ args=[ main.Cluster.active() ],
sleep=15,
attempts=5 )
@@ -185,11 +171,10 @@
onfail="Nodes check NOT successful" )
if not nodeResults:
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
+ for ctrl in main.Cluster.active():
main.log.debug( "{} components not ACTIVE: \n{}".format(
- cli.name,
- cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ ctrl.name,
+ ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -209,21 +194,18 @@
except AttributeError:
main.kill = []
-
main.HA.checkStateAfterONOS( main, afterWhich=0 )
-
main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leaderList = []
restarted = []
- for i in main.kill:
- restarted.append( main.nodes[ i ].ip_address )
+ for ctrl in main.kill:
+ restarted.append( ctrl.ipAddress )
leaderResult = main.TRUE
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- leaderN = cli.electionTestLeader()
+ for ctrl in main.Cluster.active():
+ leaderN = ctrl.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
@@ -232,12 +214,12 @@
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.error( cli.name +
+ main.log.error( ctrl.name +
" shows no leader for the election-app was" +
" elected after the old one died" )
leaderResult = main.FALSE
elif leaderN in restarted:
- main.log.error( cli.name + " shows " + str( leaderN ) +
+ main.log.error( ctrl.name + " shows " + str( leaderN ) +
" as leader for the election-app, but it " +
"was restarted" )
leaderResult = main.FALSE