Update Cluster Driver
Change-Id: I8a3a57e19637ff210548e57d41178e6f194cf694
diff --git a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
index 14ffc7c..eb7cf8a 100644
--- a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
@@ -88,18 +88,13 @@
# load some variables from the params file
cellName = main.params[ 'ENV' ][ 'cellName' ]
main.apps = main.params[ 'ENV' ][ 'appString' ]
- main.numCtrls = int( main.params[ 'num_controllers' ] )
- if main.ONOSbench.maxNodes and \
- main.ONOSbench.maxNodes < main.numCtrls:
- main.numCtrls = int( main.ONOSbench.maxNodes )
- main.maxNodes = main.numCtrls
- stepResult = main.testSetUp.envSetup( hasNode=True )
+ stepResult = main.testSetUp.envSetup()
except Exception as e:
main.testSetUp.envSetupException( e )
main.testSetUp.evnSetupConclusion( stepResult )
main.HA.generateGraph( "HAclusterRestart" )
- main.testSetUp.ONOSSetUp( main.Mininet1, cellName=cellName, removeLog=True,
+ main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
extraApply=main.HA.startingMininet )
main.HA.initialSetUp()
@@ -138,11 +133,8 @@
The Failure case.
"""
import time
- assert main.numCtrls, "main.numCtrls not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- assert main.CLIs, "main.CLIs not defined"
- assert main.nodes, "main.nodes not defined"
try:
main.HAlabels
except ( NameError, AttributeError ):
@@ -163,61 +155,21 @@
main.case( "Restart entire ONOS cluster" )
main.step( "Checking ONOS Logs for errors" )
- for node in main.nodes:
- main.log.debug( "Checking logs for errors on " + node.name + ":" )
- main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
+ for ctrl in main.Cluster.active():
+ main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
+ main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
- main.step( "Killing ONOS nodes" )
- killResults = main.TRUE
- killTime = time.time()
- for node in main.nodes:
- killed = main.ONOSbench.onosKill( node.ip_address )
- killResults = killResults and killed
- utilities.assert_equals( expect=main.TRUE, actual=killResults,
- onpass="ONOS nodes killed",
- onfail="ONOS kill unsuccessful" )
+ main.testSetUp.killingAllOnos( main.Cluster, True, False )
- main.step( "Checking if ONOS is up yet" )
- for i in range( 2 ):
- onosIsupResult = main.TRUE
- for node in main.nodes:
- started = main.ONOSbench.isup( node.ip_address )
- if not started:
- main.log.error( node.name + " didn't start!" )
- onosIsupResult = onosIsupResult and started
- if onosIsupResult == main.TRUE:
- break
- utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
- onpass="ONOS restarted",
- onfail="ONOS restart NOT successful" )
+ main.testSetUp.checkOnosService( main.Cluster )
- main.step( "Starting ONOS CLI sessions" )
- cliResults = main.TRUE
- threads = []
- for i in range( main.numCtrls ):
- t = main.Thread( target=main.CLIs[ i ].startOnosCli,
- name="startOnosCli-" + str( i ),
- args=[ main.nodes[ i ].ip_address ] )
- threads.append( t )
- t.start()
+ main.testSetUp.startOnosClis( main.Cluster )
- for t in threads:
- t.join()
- cliResults = cliResults and t.result
- utilities.assert_equals( expect=main.TRUE, actual=cliResults,
- onpass="ONOS cli started",
- onfail="ONOS clis did not restart" )
-
- for i in range( 10 ):
- ready = True
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- output = cli.summary()
- if not output:
- ready = False
- if ready:
- break
- time.sleep( 30 )
+ ready = utilities.retry( main.Cluster.command,
+ False,
+ kwargs={ "function":"summary", "contentCheck":True },
+ sleep=30,
+ attempts=10 )
utilities.assert_equals( expect=True, actual=ready,
onpass="ONOS summary command succeded",
onfail="ONOS summary command failed" )
@@ -233,23 +185,15 @@
main.HAdata.append( str( main.restartTime ) )
# Rerun for election on restarted nodes
- runResults = main.TRUE
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- run = cli.electionTestRun()
- if run != main.TRUE:
- main.log.error( "Error running for election on " + cli.name )
- runResults = runResults and run
- utilities.assert_equals( expect=main.TRUE, actual=runResults,
+ runResults = main.Cluster.command( "electionTestRun", returnBool=True )
+ utilities.assert_equals( expect=True, actual=runResults,
onpass="Reran for election",
onfail="Failed to rerun for election" )
# TODO: Make this configurable
time.sleep( 60 )
- node = main.activeNodes[ 0 ]
- main.log.debug( main.CLIs[ node ].nodes( jsonFormat=False ) )
- main.log.debug( main.CLIs[ node ].leaders( jsonFormat=False ) )
- main.log.debug( main.CLIs[ node ].partitions( jsonFormat=False ) )
+
+ main.HA.commonChecks()
def CASE7( self, main ):
"""
@@ -257,16 +201,16 @@
"""
# NOTE: Store has no durability, so intents are lost across system
# restarts
- main.HA.checkStateAfterONOS( main, afterWhich=0, isRestart=True )
+ main.HA.checkStateAfterEvent( main, afterWhich=0, isRestart=True )
main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leaderList = []
leaderResult = main.TRUE
- for i in main.activeNodes:
- cli = main.CLIs[ i ]
- leaderN = cli.electionTestLeader()
+ for ctrl in main.Cluster.active():
+ ctrl.CLI.electionTestLeader()
+ leaderN = ctrl.CLI.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
diff --git a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.topo b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.topo
index 7c18a98..4bf4bd4 100644
--- a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.topo
+++ b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.topo
@@ -1,183 +1,32 @@
<TOPOLOGY>
<COMPONENT>
- <ONOSbench>
- <host>localhost</host>
+ <ONOScell>
+ <host>localhost</host> # ONOS "bench" machine
<user>sdn</user>
<password>rocks</password>
- <type>OnosDriver</type>
+ <type>OnosClusterDriver</type>
<connect_order>1</connect_order>
<COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOSbench>
-
- <ONOScli1>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>2</connect_order>
- <COMPONENTS>
+ <cluster_name></cluster_name> # Used as a prefix for cluster components. Defaults to 'ONOS'
+ <diff_clihost></diff_clihost> # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
<karaf_username></karaf_username>
<karaf_password></karaf_password>
- <prompt></prompt>
+ <web_user></web_user>
+ <web_pass></web_pass>
+ <rest_port></rest_port>
+ <prompt></prompt> # TODO: we technically need a few of these, one per component
+ <onos_home></onos_home> # defines where onos home is
+ <nodes> 7 </nodes> # number of nodes in the cluster
</COMPONENTS>
- </ONOScli1>
-
- <ONOScli2>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>3</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli2>
-
- <ONOScli3>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>4</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli3>
-
-
- <ONOScli4>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>5</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli4>
-
-
- <ONOScli5>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>6</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli5>
-
-
- <ONOScli6>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>7</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli6>
-
-
- <ONOScli7>
- <host>localhost</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosCliDriver</type>
- <connect_order>8</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOScli7>
-
- <ONOS1>
- <host>OC1</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>9</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS1>
-
- <ONOS2>
- <host>OC2</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>10</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS2>
-
- <ONOS3>
- <host>OC3</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>11</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS3>
-
- <ONOS4>
- <host>OC4</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>12</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS4>
-
- <ONOS5>
- <host>OC5</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>13</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS5>
-
- <ONOS6>
- <host>OC6</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>14</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS6>
-
- <ONOS7>
- <host>OC7</host>
- <user>sdn</user>
- <password>rocks</password>
- <type>OnosDriver</type>
- <connect_order>15</connect_order>
- <COMPONENTS>
- <prompt></prompt>
- </COMPONENTS>
- </ONOS7>
+ </ONOScell>
<Mininet1>
<host>OCN</host>
<user>sdn</user>
<password>rocks</password>
<type>MininetCliDriver</type>
- <connect_order>16</connect_order>
+ <connect_order>2</connect_order>
<COMPONENTS>
#Specify the Option for mininet
<arg1> --custom ~/mininet/custom/obelisk.py </arg1>
@@ -194,7 +43,7 @@
<user>sdn</user>
<password>rocks</password>
<type>RemoteMininetDriver</type>
- <connect_order>17</connect_order>
+ <connect_order>3</connect_order>
<COMPONENTS>
<prompt></prompt>
</COMPONENTS>