Refactor HAminorityRestart
- Split into two tests, one that kills ONOS and one that stops the
service
- Check ONOS between brining some ONOS nodes down and up
- Set "gossip" time to be a function of # of nodes
Change-Id: I389164c8225b2e496db433552a7ba5a5a3e5c2bb
diff --git a/TestON/tests/HAkillNodes/dependencies/Counters.py b/TestON/tests/HAkillNodes/dependencies/Counters.py
new file mode 100644
index 0000000..455e3e7
--- /dev/null
+++ b/TestON/tests/HAkillNodes/dependencies/Counters.py
@@ -0,0 +1,103 @@
+def __init__( self ):
+ self.default = ''
+
+def consistentCheck():
+ """
+ Checks that TestON counters are consistent across all nodes.
+
+ Returns the tuple (onosCounters, consistent)
+ - onosCounters is the parsed json output of the counters command on all nodes
+ - consistent is main.TRUE if all "TestON" counters are consitent across all
+ nodes or main.FALSE
+ """
+ import json
+ try:
+ correctResults = main.TRUE
+ # Get onos counters results
+ onosCountersRaw = []
+ threads = []
+ for i in main.activeNodes:
+ t = main.Thread( target=main.CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCountersRaw.append( t.result )
+ onosCounters = []
+ for i in range( len( main.activeNodes ) ):
+ try:
+ onosCounters.append( json.loads( onosCountersRaw[i] ) )
+ except ( ValueError, TypeError ):
+ main.log.error( "Could not parse counters response from ONOS" +
+ str( main.activeNodes[i] + 1 ) )
+ main.log.warn( repr( onosCountersRaw[ i ] ) )
+ return main.FALSE
+
+ testCounters = {}
+ # make a list of all the "TestON-*" counters in ONOS
+ # lookes like a dict whose keys are the name of the ONOS node and values
+ # are a list of the counters. I.E.
+ # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
+ # {"name":"TestON-Partitions","value":56} ]
+ # }
+ # NOTE: There is an assumtion that all nodes are active
+ # based on the above for loops
+ for controller in enumerate( onosCounters ):
+ for dbType in controller[1]:
+ for dbName, items in dbType.iteritems():
+ for item in items:
+ if 'TestON' in item['name']:
+ node = 'ONOS' + str( main.activeNodes[ controller[0] ] + 1 )
+ try:
+ testCounters[node].append( item )
+ except KeyError:
+ testCounters[node] = [ item ]
+ # compare the counters on each node
+ firstV = testCounters.values()[0]
+ tmp = [ v == firstV for k, v in testCounters.iteritems() ]
+ if all( tmp ):
+ consistent = main.TRUE
+ else:
+ consistent = main.FALSE
+ main.log.error( "ONOS nodes have different values for counters:\n" +
+ testCounters )
+ return ( onosCounters, consistent )
+ except Exception:
+ main.log.exception( "" )
+ main.cleanup()
+ main.exit()
+
+def counterCheck( counterName, counterValue ):
+ """
+ Checks that TestON counters are consistent across all nodes and that
+ specified counter is in ONOS with the given value
+ """
+ import json
+ correctResults = main.TRUE
+ # Get onos counters results and consistentCheck
+ onosCounters, consistent = main.Counters.consistentCheck()
+ # Check for correct values
+ for i in range( len( main.activeNodes ) ):
+ current = onosCounters[i]
+ onosValue = None
+ try:
+ for database in current:
+ database = database.values()[0]
+ for counter in database:
+ if counter.get( 'name' ) == counterName:
+ onosValue = counter.get( 'value' )
+ break
+ except AttributeError, e:
+ node = str( main.activeNodes[i] + 1 )
+ main.log.error( "ONOS" + node + " counters result " +
+ "is not as expected" )
+ correctResults = main.FALSE
+ if onosValue == counterValue:
+ main.log.info( counterName + " counter value is correct" )
+ else:
+ main.log.error( counterName + " counter value is incorrect," +
+ " expected value: " + str( counterValue )
+ + " current value: " + str( onosValue ) )
+ correctResults = main.FALSE
+ return consistent and correctResults
diff --git a/TestON/tests/HAkillNodes/dependencies/__init__.py b/TestON/tests/HAkillNodes/dependencies/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HAkillNodes/dependencies/__init__.py
diff --git a/TestON/tests/HAkillNodes/dependencies/obelisk.py b/TestON/tests/HAkillNodes/dependencies/obelisk.py
new file mode 100755
index 0000000..d613806
--- /dev/null
+++ b/TestON/tests/HAkillNodes/dependencies/obelisk.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+from mininet.topo import Topo
+
+class ObeliskTopo( Topo ):
+ def __init__( self ):
+ Topo.__init__( self )
+ topSwitch = self.addSwitch('s1',dpid='1000'.zfill(16))
+ leftTopSwitch = self.addSwitch('s2',dpid='2000'.zfill(16))
+ rightTopSwitch = self.addSwitch('s5',dpid='5000'.zfill(16))
+ leftBotSwitch = self.addSwitch('s3',dpid='3000'.zfill(16))
+ rightBotSwitch = self.addSwitch('s6',dpid='6000'.zfill(16))
+ midBotSwitch = self.addSwitch('s28',dpid='2800'.zfill(16))
+
+ topHost = self.addHost( 'h1' )
+ leftTopHost = self.addHost('h2')
+ rightTopHost = self.addHost('h5')
+ leftBotHost = self.addHost('h3')
+ rightBotHost = self.addHost('h6')
+ midBotHost = self.addHost('h28')
+ self.addLink(topSwitch,topHost)
+ self.addLink(leftTopSwitch,leftTopHost)
+ self.addLink(rightTopSwitch,rightTopHost)
+ self.addLink(leftBotSwitch,leftBotHost)
+ self.addLink(rightBotSwitch,rightBotHost)
+ self.addLink(midBotSwitch,midBotHost)
+ self.addLink(leftTopSwitch,rightTopSwitch)
+ self.addLink(topSwitch,leftTopSwitch)
+ self.addLink(topSwitch,rightTopSwitch)
+ self.addLink(leftTopSwitch,leftBotSwitch)
+ self.addLink(rightTopSwitch,rightBotSwitch)
+ self.addLink(leftBotSwitch,midBotSwitch)
+ self.addLink(midBotSwitch,rightBotSwitch)
+
+ agg1Switch = self.addSwitch('s4',dpid = '3004'.zfill(16))
+ agg2Switch = self.addSwitch('s7',dpid = '6007'.zfill(16))
+ agg1Host = self.addHost('h4')
+ agg2Host = self.addHost('h7')
+ self.addLink(agg1Switch,agg1Host)
+ self.addLink(agg2Switch,agg2Host)
+ self.addLink(agg1Switch, leftBotSwitch)
+ self.addLink(agg2Switch, rightBotSwitch)
+
+ for i in range(10):
+ num = str(i+8)
+ switch = self.addSwitch('s'+num,dpid = ('30'+num.zfill(2)).zfill(16))
+ host = self.addHost('h'+num)
+ self.addLink(switch, host)
+ self.addLink(switch, agg1Switch)
+
+ for i in range(10):
+ num = str(i+18)
+ switch = self.addSwitch('s'+num,dpid = ('60'+num.zfill(2)).zfill(16))
+ host = self.addHost('h'+num)
+ self.addLink(switch, host)
+ self.addLink(switch, agg2Switch)
+
+topos = { 'obelisk': (lambda: ObeliskTopo() ) }
+
+def run():
+ topo = ObeliskTopo()
+ net = Mininet( topo=topo, controller=RemoteController, autoSetMacs=True )
+ net.start()
+ CLI( net )
+ net.stop()
+
+if __name__ == '__main__':
+ setLogLevel( 'info' )
+ run()
diff --git a/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions b/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
new file mode 100755
index 0000000..b73ed4a
--- /dev/null
+++ b/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+'''
+ Generate the partitions json file from the $OC* environment variables
+
+ Usage: onos-gen-partitions [output file]
+ If output file is not provided, the json is written to stdout.
+'''
+
+from os import environ
+from collections import deque, OrderedDict
+import re
+import json
+import sys
+
+convert = lambda text: int(text) if text.isdigit() else text.lower()
+alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
+
+def get_OC_vars():
+ vars = []
+ for var in environ:
+ if re.match(r"OC[0-9]+", var):
+ vars.append(var)
+ return sorted(vars, key=alphanum_key)
+
+def get_nodes(vars, port=9876):
+ node = lambda k: { 'id': k, 'ip': k, 'port': port }
+ return [ node(environ[v]) for v in vars ]
+
+def generate_permutations(nodes, k):
+ l = deque(nodes)
+ perms = []
+ for i in range(1, len(nodes)+1):
+ part = {
+ 'name': 'p%d' % i,
+ 'members': list(l)[:k]
+ }
+ perms.append(part)
+ l.rotate(-1)
+ return perms
+
+def generate_permutations2(nodes, k):
+ l = deque(nodes)
+ perms = []
+ for i in range(1, (len(nodes) + 1) / 2 + 1):
+ part = {
+ 'name': 'p%d' % i,
+ 'members': list(l)[:k]
+ }
+ perms.append(part)
+ l.rotate(-2)
+ return perms
+
+if __name__ == '__main__':
+ vars = get_OC_vars()
+ nodes = get_nodes(vars)
+ partitions = generate_permutations2([v.get('id') for v in nodes], 3)
+ data = {
+ 'name': 'default',
+ 'nodes': nodes,
+ 'partitions': partitions
+ }
+ output = json.dumps(data, indent=4)
+
+ if len(sys.argv) == 2:
+ filename = sys.argv[1]
+ with open(filename, 'w') as f:
+ f.write(output)
+ else:
+ print output