Update Cluster Driver
Change-Id: I8a3a57e19637ff210548e57d41178e6f194cf694
diff --git a/TestON/tests/HA/HAscaling/HAscaling.py b/TestON/tests/HA/HAscaling/HAscaling.py
index f35fa57..6a0abd4 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.py
+++ b/TestON/tests/HA/HAscaling/HAscaling.py
@@ -92,20 +92,17 @@
# load some variables from the params file
cellName = main.params[ 'ENV' ][ 'cellName' ]
main.apps = main.params[ 'ENV' ][ 'appString' ]
- main.numCtrls = int( main.params[ 'num_controllers' ] )
- if main.ONOSbench.maxNodes and\
- main.ONOSbench.maxNodes < main.numCtrls:
- main.numCtrls = int( main.ONOSbench.maxNodes )
- main.maxNodes = main.numCtrls
- stepResult = main.testSetUp.envSetup( hasNode=True )
+ stepResult = main.testSetUp.envSetup()
except Exception as e:
main.testSetUp.envSetupException( e )
main.testSetUp.evnSetupConclusion( stepResult )
main.HA.generateGraph( "HAscaling", index=1 )
- main.testSetUp.ONOSSetUp( main.Mininet1, cellName=cellName, removeLog=True,
- extraApply=main.HA.customizeOnosService,
- arg=main.HA.scalingMetadata,
+ main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
+ extraApply=[ main.HA.setServerForCluster,
+ main.HA.scalingMetadata,
+ main.HA.startingMininet,
+ main.HA.copyingBackupConfig ],
extraClean=main.HA.cleanUpOnosService,
installMax=True )
@@ -147,11 +144,8 @@
"""
import time
import re
- assert main.numCtrls, "main.numCtrls not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- assert main.CLIs, "main.CLIs not defined"
- assert main.nodes, "main.nodes not defined"
try:
main.HAlabels
except ( NameError, AttributeError ):
@@ -166,10 +160,9 @@
main.case( "Scale the number of nodes in the ONOS cluster" )
main.step( "Checking ONOS Logs for errors" )
- for i in main.activeNodes:
- node = main.nodes[ i ]
- main.log.debug( "Checking logs for errors on " + node.name + ":" )
- main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
+ for ctrl in main.Cluster.active():
+ main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
+ main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
"""
pop # of nodes from a list, might look like 1,3b,3,5b,5,7b,7,7b,5,5b,3...
@@ -177,15 +170,15 @@
install/deactivate node as needed
"""
try:
- prevNodes = main.activeNodes
+ prevNodes = main.Cluster.active()
scale = main.scaling.pop( 0 )
if "e" in scale:
equal = True
else:
equal = False
- main.numCtrls = int( re.search( "\d+", scale ).group( 0 ) )
- main.log.info( "Scaling to {} nodes".format( main.numCtrls ) )
- genResult = main.Server.generateFile( main.numCtrls, equal=equal )
+ main.Cluster.setRunningNode( int( re.search( "\d+", scale ).group( 0 ) ) )
+ main.log.info( "Scaling to {} nodes".format( main.Cluster.numCtrls ) )
+ genResult = main.Server.generateFile( main.Cluster.numCtrls, equal=equal )
utilities.assert_equals( expect=main.TRUE, actual=genResult,
onpass="New cluster metadata file generated",
onfail="Failled to generate new metadata file" )
@@ -194,54 +187,27 @@
main.cleanup()
main.exit()
- main.activeNodes = [ i for i in range( 0, main.numCtrls ) ]
- newNodes = [ x for x in main.activeNodes if x not in prevNodes ]
-
+ activeNodes = [ i for i in range( 0, main.Cluster.numCtrls ) ]
+ newNodes = [ x for x in activeNodes if x not in prevNodes ]
+ main.Cluster.resetActive()
main.step( "Start new nodes" ) # OR stop old nodes?
started = main.TRUE
for i in newNodes:
- started = main.ONOSbench.onosStart( main.nodes[ i ].ip_address ) and main.TRUE
+ started = main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) and main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=started,
onpass="ONOS started",
onfail="ONOS start NOT successful" )
- main.step( "Checking if ONOS is up yet" )
- for i in range( 2 ):
- onosIsupResult = main.TRUE
- for i in main.activeNodes:
- node = main.nodes[ i ]
- main.ONOSbench.onosSecureSSH( node=node.ip_address )
- started = main.ONOSbench.isup( node.ip_address )
- if not started:
- main.log.error( node.name + " didn't start!" )
- onosIsupResult = onosIsupResult and started
- if onosIsupResult == main.TRUE:
- break
- utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
- onpass="ONOS started",
- onfail="ONOS start NOT successful" )
+ main.testSetUp.setupSsh( main.Cluster )
- main.step( "Starting ONOS CLI sessions" )
- cliResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[ i ].startOnosCli,
- name="startOnosCli-" + str( i ),
- args=[ main.nodes[ i ].ip_address ] )
- threads.append( t )
- t.start()
+ main.testSetUp.checkOnosService( main.Cluster )
- for t in threads:
- t.join()
- cliResults = cliResults and t.result
- utilities.assert_equals( expect=main.TRUE, actual=cliResults,
- onpass="ONOS cli started",
- onfail="ONOS clis did not start" )
+ main.Cluster.startCLIs()
main.step( "Checking ONOS nodes" )
nodeResults = utilities.retry( main.HA.nodesCheck,
False,
- args=[ main.activeNodes ],
+ args=[ main.Cluster.active() ],
attempts=5 )
utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
@@ -249,9 +215,8 @@
for i in range( 10 ):
ready = True
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- output = cli.summary()
+ for ctrl in main.Cluster.active():
+ output = ctrl.CLI.summary()
if not output:
ready = False
if ready:
@@ -266,11 +231,10 @@
# Rerun for election on new nodes
runResults = main.TRUE
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- run = cli.electionTestRun()
+ for ctrl in main.Cluster.active():
+ run = ctrl.CLI.electionTestRun()
if run != main.TRUE:
- main.log.error( "Error running for election on " + cli.name )
+ main.log.error( "Error running for election on " + ctrl.name )
runResults = runResults and run
utilities.assert_equals( expect=main.TRUE, actual=runResults,
onpass="Reran for election",
@@ -278,28 +242,22 @@
# TODO: Make this configurable
time.sleep( 60 )
- for node in main.activeNodes:
- main.log.warn( "\n****************** {} **************".format( main.nodes[ node ].ip_address ) )
- main.log.debug( main.CLIs[ node ].nodes( jsonFormat=False ) )
- main.log.debug( main.CLIs[ node ].leaders( jsonFormat=False ) )
- main.log.debug( main.CLIs[ node ].partitions( jsonFormat=False ) )
- main.log.debug( main.CLIs[ node ].apps( jsonFormat=False ) )
+ main.HA.commonChecks()
def CASE7( self, main ):
"""
Check state after ONOS scaling
"""
- main.HA.checkStateAfterONOS( main, afterWhich=1 )
+ main.HA.checkStateAfterEvent( main, afterWhich=1 )
main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leaderList = []
leaderResult = main.TRUE
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- leaderN = cli.electionTestLeader()
+ for ctrl in main.Cluster.active():
+ leaderN = ctrl.CLI.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
diff --git a/TestON/tests/HA/HAscaling/HAscaling.topo b/TestON/tests/HA/HAscaling/HAscaling.topo
index 7c18a98..4bf4bd4 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.topo
+++ b/TestON/tests/HA/HAscaling/HAscaling.topo
@@ -1,183 +1,32 @@
<TOPOLOGY>
<COMPONENT>
- <ONOSbench>
- <host>localhost</host>
+ <ONOScell>
+ <host>localhost</host> # ONOS "bench" machine
<user>sdn</user>
<password>rocks</password>
- <type>OnosDriver</type>
+ <type>OnosClusterDriver</type>
<connect_order>1</connect_order>
<COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOSbench>
-
- <ONOScli1>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>2</connect_order>
- <COMPONENTS>
+ <cluster_name></cluster_name> # Used as a prefix for cluster components. Defaults to 'ONOS'
+ <diff_clihost></diff_clihost> # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
<karaf_username></karaf_username>
<karaf_password></karaf_password>
- <prompt></prompt>
+ <web_user></web_user>
+ <web_pass></web_pass>
+ <rest_port></rest_port>
+ <prompt></prompt> # TODO: we technically need a few of these, one per component
+ <onos_home></onos_home> # defines where onos home is
+ <nodes> 7 </nodes> # number of nodes in the cluster
</COMPONENTS>
- </ONOScli1>
-
- <ONOScli2>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>3</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli2>
-
- <ONOScli3>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>4</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli3>
-
-
- <ONOScli4>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>5</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli4>
-
-
- <ONOScli5>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>6</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli5>
-
-
- <ONOScli6>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>7</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli6>
-
-
- <ONOScli7>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>8</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli7>
-
- <ONOS1>
- <host>OC1</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>9</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS1>
-
- <ONOS2>
- <host>OC2</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>10</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS2>
-
- <ONOS3>
- <host>OC3</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>11</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS3>
-
- <ONOS4>
- <host>OC4</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>12</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS4>
-
- <ONOS5>
- <host>OC5</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>13</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS5>
-
- <ONOS6>
- <host>OC6</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>14</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS6>
-
- <ONOS7>
- <host>OC7</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>15</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS7>
+ </ONOScell>
<Mininet1>
<host>OCN</host>
<user>sdn</user>
<password>rocks</password>
<type>MininetCliDriver</type>
- <connect_order>16</connect_order>
+ <connect_order>2</connect_order>
<COMPONENTS>
#Specify the Option for mininet
<arg1> --custom ~/mininet/custom/obelisk.py </arg1>
@@ -194,7 +43,7 @@
<user>sdn</user>
<password>rocks</password>
<type>RemoteMininetDriver</type>
- <connect_order>17</connect_order>
+ <connect_order>3</connect_order>
<COMPONENTS>
<prompt></prompt>
</COMPONENTS>