Update Cluster Driver
Change-Id: I8a3a57e19637ff210548e57d41178e6f194cf694
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.py b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
index c1e3006..923f67c 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.py
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
@@ -90,21 +90,18 @@
# load some variables from the params file
cellName = main.params[ 'ENV' ][ 'cellName' ]
main.apps = main.params[ 'ENV' ][ 'appString' ]
- main.numCtrls = int( main.params[ 'num_controllers' ] )
- if main.ONOSbench.maxNodes and\
- main.ONOSbench.maxNodes < main.numCtrls:
- main.numCtrls = int( main.ONOSbench.maxNodes )
- main.maxNodes = main.numCtrls
- stepResult = main.testSetUp.envSetup( hasNode=True )
+ stepResult = main.testSetUp.envSetup()
except Exception as e:
main.testSetUp.envSetupException( e )
main.testSetUp.evnSetupConclusion( stepResult )
main.HA.generateGraph( "HAswapNodes" )
- main.testSetUp.ONOSSetUp( main.Mininet1, cellName=cellName, removeLog=True,
- extraApply=main.HA.customizeOnosService,
- arg=main.HA.swapNodeMetadata,
+ main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
+ extraApply=[ main.HA.setServerForCluster,
+ main.HA.swapNodeMetadata,
+ main.HA.startingMininet,
+ main.HA.copyingBackupConfig ],
extraClean=main.HA.cleanUpOnosService,
installMax=True )
main.HA.initialSetUp()
@@ -145,11 +142,8 @@
"""
import time
import re
- assert main.numCtrls, "main.numCtrls not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- assert main.CLIs, "main.CLIs not defined"
- assert main.nodes, "main.nodes not defined"
try:
main.HAlabels
except ( NameError, AttributeError ):
@@ -164,14 +158,15 @@
main.case( "Swap some of the ONOS nodes" )
main.step( "Checking ONOS Logs for errors" )
- for i in main.activeNodes:
- node = main.nodes[ i ]
- main.log.debug( "Checking logs for errors on " + node.name + ":" )
- main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
+ for ctrl in main.Cluster.active():
+ main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
+ main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
+ activeNodes = main.Cluster.getRunningPos()
+ # Todo : this could be wrong. need to double check.
main.step( "Generate new metadata file" )
- old = [ main.activeNodes[ 1 ], main.activeNodes[ -2 ] ]
- new = range( main.ONOSbench.maxNodes )[ -2: ]
+ old = [ activeNodes[ 1 ], activeNodes[ -2 ] ]
+ new = range( main.Cluster.maxCtrls )[ -2: ]
assert len( old ) == len( new ), "Length of nodes to swap don't match"
handle = main.ONOSbench.handle
for x, y in zip( old, new ):
@@ -181,75 +176,45 @@
handle.expect( "\$" ) # From the prompt
ret += handle.before
main.log.debug( ret )
- main.activeNodes.remove( x )
- main.activeNodes.append( y )
+ activeNodes.remove( x )
+ activeNodes.append( y )
- genResult = main.Server.generateFile( main.numCtrls )
+ genResult = main.Server.generateFile( main.Cluster.numCtrls )
utilities.assert_equals( expect=main.TRUE, actual=genResult,
onpass="New cluster metadata file generated",
onfail="Failled to generate new metadata file" )
time.sleep( 5 ) # Give time for nodes to read new file
-
+ main.Cluster.resetActive()
+ # Note : done up to this point.
main.step( "Start new nodes" ) # OR stop old nodes?
started = main.TRUE
for i in new:
- started = main.ONOSbench.onosStart( main.nodes[ i ].ip_address ) and main.TRUE
+ started = main.ONOSbench.onosStart( main.Cluster.controllers[ i ].ipAddress ) and main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=started,
onpass="ONOS started",
onfail="ONOS start NOT successful" )
- main.step( "Checking if ONOS is up yet" )
- for i in range( 2 ):
- onosIsupResult = main.TRUE
- for i in main.activeNodes:
- node = main.nodes[ i ]
- main.ONOSbench.onosSecureSSH( node=node.ip_address )
- started = main.ONOSbench.isup( node.ip_address )
- if not started:
- main.log.error( node.name + " didn't start!" )
- onosIsupResult = onosIsupResult and started
- if onosIsupResult == main.TRUE:
- break
- utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
- onpass="ONOS started",
- onfail="ONOS start NOT successful" )
+ main.Cluster.setRunningNode( activeNodes )
- main.step( "Starting ONOS CLI sessions" )
- cliResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[ i ].startOnosCli,
- name="startOnosCli-" + str( i ),
- args=[ main.nodes[ i ].ip_address ] )
- threads.append( t )
- t.start()
+ main.testSetUp.setupSsh( main.Cluster )
+ main.testSetUp.checkOnosService( main.Cluster )
- for t in threads:
- t.join()
- cliResults = cliResults and t.result
- utilities.assert_equals( expect=main.TRUE, actual=cliResults,
- onpass="ONOS cli started",
- onfail="ONOS clis did not start" )
+ main.testSetUp.startOnosClis( main.Cluster )
main.step( "Checking ONOS nodes" )
nodeResults = utilities.retry( main.HA.nodesCheck,
False,
- args=[ main.activeNodes ],
+ args=[ main.Cluster.active() ],
attempts=5 )
utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
- for i in range( 10 ):
- ready = True
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- output = cli.summary()
- if not output:
- ready = False
- if ready:
- break
- time.sleep( 30 )
+ ready = utilities.retry( main.Cluster.command,
+ False,
+ kwargs={ "function":"summary", "contentCheck":True },
+ sleep=30,
+ attempts=10 )
utilities.assert_equals( expect=True, actual=ready,
onpass="ONOS summary command succeded",
onfail="ONOS summary command failed" )
@@ -259,22 +224,19 @@
# Rerun for election on new nodes
runResults = main.TRUE
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- run = cli.electionTestRun()
+ for ctrl in main.Cluster.active():
+ run = ctrl.CLI.electionTestRun()
if run != main.TRUE:
- main.log.error( "Error running for election on " + cli.name )
+ main.log.error( "Error running for election on " + ctrl.name )
runResults = runResults and run
utilities.assert_equals( expect=main.TRUE, actual=runResults,
onpass="Reran for election",
onfail="Failed to rerun for election" )
- for node in main.activeNodes:
- main.log.warn( "\n****************** {} **************".format( main.nodes[ node ].ip_address ) )
- main.log.debug( main.CLIs[ node ].nodes( jsonFormat=False ) )
- main.log.debug( main.CLIs[ node ].leaders( jsonFormat=False ) )
- main.log.debug( main.CLIs[ node ].partitions( jsonFormat=False ) )
- main.log.debug( main.CLIs[ node ].apps( jsonFormat=False ) )
+ main.HA.commonChecks()
+
+ """
+ # Note: Do we really want this? It will revert the changes we have made from this test case.
main.step( "Reapplying cell variable to environment" )
cellName = main.params[ 'ENV' ][ 'cellName' ]
@@ -282,22 +244,23 @@
utilities.assert_equals( expect=main.TRUE, actual=cellResult,
onpass="Set cell successfull",
onfail="Failled to set cell" )
+ """
+
def CASE7( self, main ):
"""
Check state after ONOS scaling
"""
- main.HA.checkStateAfterONOS( main, afterWhich=1 )
+ main.HA.checkStateAfterEvent( main, afterWhich=1 )
main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leaderList = []
leaderResult = main.TRUE
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- leaderN = cli.electionTestLeader()
+ for ctrl in main.Cluster.active():
+ leaderN = ctrl.CLI.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
@@ -306,7 +269,7 @@
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.error( cli.name +
+ main.log.error( ctrl.name +
" shows no leader for the election-app." )
leaderResult = main.FALSE
if len( set( leaderList ) ) != 1:
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.topo b/TestON/tests/HA/HAswapNodes/HAswapNodes.topo
index 7c18a98..4bf4bd4 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.topo
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.topo
@@ -1,183 +1,32 @@
<TOPOLOGY>
<COMPONENT>
- <ONOSbench>
- <host>localhost</host>
+ <ONOScell>
+ <host>localhost</host> # ONOS "bench" machine
<user>sdn</user>
<password>rocks</password>
- <type>OnosDriver</type>
+ <type>OnosClusterDriver</type>
<connect_order>1</connect_order>
<COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOSbench>
-
- <ONOScli1>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>2</connect_order>
- <COMPONENTS>
+ <cluster_name></cluster_name> # Used as a prefix for cluster components. Defaults to 'ONOS'
+ <diff_clihost></diff_clihost> # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
<karaf_username></karaf_username>
<karaf_password></karaf_password>
- <prompt></prompt>
+ <web_user></web_user>
+ <web_pass></web_pass>
+ <rest_port></rest_port>
+ <prompt></prompt> # TODO: we technically need a few of these, one per component
+ <onos_home></onos_home> # defines where onos home is
+ <nodes> 7 </nodes> # number of nodes in the cluster
</COMPONENTS>
- </ONOScli1>
-
- <ONOScli2>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>3</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli2>
-
- <ONOScli3>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>4</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli3>
-
-
- <ONOScli4>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>5</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli4>
-
-
- <ONOScli5>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>6</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli5>
-
-
- <ONOScli6>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>7</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli6>
-
-
- <ONOScli7>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>8</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli7>
-
- <ONOS1>
- <host>OC1</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>9</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS1>
-
- <ONOS2>
- <host>OC2</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>10</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS2>
-
- <ONOS3>
- <host>OC3</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>11</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS3>
-
- <ONOS4>
- <host>OC4</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>12</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS4>
-
- <ONOS5>
- <host>OC5</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>13</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS5>
-
- <ONOS6>
- <host>OC6</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>14</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS6>
-
- <ONOS7>
- <host>OC7</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>15</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS7>
+ </ONOScell>
<Mininet1>
<host>OCN</host>
<user>sdn</user>
<password>rocks</password>
<type>MininetCliDriver</type>
- <connect_order>16</connect_order>
+ <connect_order>2</connect_order>
<COMPONENTS>
#Specify the Option for mininet
<arg1> --custom ~/mininet/custom/obelisk.py </arg1>
@@ -194,7 +43,7 @@
<user>sdn</user>
<password>rocks</password>
<type>RemoteMininetDriver</type>
- <connect_order>17</connect_order>
+ <connect_order>3</connect_order>
<COMPONENTS>
<prompt></prompt>
</COMPONENTS>