Refactor HAminorityRestart

- Split into two tests, one that kills ONOS and one that stops the
  service
- Check ONOS between brining some ONOS nodes down and up
- Set "gossip" time to be a function of # of nodes

Change-Id: I389164c8225b2e496db433552a7ba5a5a3e5c2bb
diff --git a/TestON/tests/HAkillNodes/dependencies/Counters.py b/TestON/tests/HAkillNodes/dependencies/Counters.py
new file mode 100644
index 0000000..455e3e7
--- /dev/null
+++ b/TestON/tests/HAkillNodes/dependencies/Counters.py
@@ -0,0 +1,103 @@
+def __init__( self ):
+    self.default = ''
+
+def consistentCheck():
+    """
+    Checks that TestON counters are consistent across all nodes.
+
+    Returns the tuple (onosCounters, consistent)
+    - onosCounters is the parsed json output of the counters command on all nodes
+    - consistent is main.TRUE if all "TestON" counters are consitent across all
+        nodes or main.FALSE
+    """
+    import json
+    try:
+        correctResults = main.TRUE
+        # Get onos counters results
+        onosCountersRaw = []
+        threads = []
+        for i in main.activeNodes:
+            t = main.Thread( target=main.CLIs[i].counters,
+                             name="counters-" + str( i ) )
+            threads.append( t )
+            t.start()
+        for t in threads:
+            t.join()
+            onosCountersRaw.append( t.result )
+        onosCounters = []
+        for i in range( len( main.activeNodes ) ):
+            try:
+                onosCounters.append( json.loads( onosCountersRaw[i] ) )
+            except ( ValueError, TypeError ):
+                main.log.error( "Could not parse counters response from ONOS" +
+                                str( main.activeNodes[i] + 1 ) )
+                main.log.warn( repr( onosCountersRaw[ i ] ) )
+                return main.FALSE
+
+        testCounters = {}
+        # make a list of all the "TestON-*" counters in ONOS
+        # lookes like a dict whose keys are the name of the ONOS node and values
+        # are a list of the counters. I.E.
+        # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
+        #              {"name":"TestON-Partitions","value":56} ]
+        # }
+        # NOTE: There is an assumtion that all nodes are active
+        #        based on the above for loops
+        for controller in enumerate( onosCounters ):
+            for dbType in controller[1]:
+                for dbName, items in dbType.iteritems():
+                    for item in items:
+                        if 'TestON' in item['name']:
+                            node = 'ONOS' + str( main.activeNodes[ controller[0] ] + 1 )
+                            try:
+                                testCounters[node].append( item )
+                            except KeyError:
+                                testCounters[node] = [ item ]
+        # compare the counters on each node
+        firstV = testCounters.values()[0]
+        tmp = [ v == firstV for k, v in testCounters.iteritems() ]
+        if all( tmp ):
+            consistent = main.TRUE
+        else:
+            consistent = main.FALSE
+            main.log.error( "ONOS nodes have different values for counters:\n" +
+                            testCounters )
+        return ( onosCounters, consistent )
+    except Exception:
+        main.log.exception( "" )
+        main.cleanup()
+        main.exit()
+
+def counterCheck( counterName, counterValue ):
+    """
+    Checks that TestON counters are consistent across all nodes and that
+    specified counter is in ONOS with the given value
+    """
+    import json
+    correctResults = main.TRUE
+    # Get onos counters results and consistentCheck
+    onosCounters, consistent = main.Counters.consistentCheck()
+    # Check for correct values
+    for i in range( len( main.activeNodes ) ):
+        current = onosCounters[i]
+        onosValue = None
+        try:
+            for database in current:
+                database = database.values()[0]
+                for counter in database:
+                    if counter.get( 'name' ) == counterName:
+                        onosValue = counter.get( 'value' )
+                        break
+        except AttributeError, e:
+            node = str( main.activeNodes[i] + 1 )
+            main.log.error( "ONOS" + node + " counters result " +
+                            "is not as expected" )
+            correctResults = main.FALSE
+        if onosValue == counterValue:
+            main.log.info( counterName + " counter value is correct" )
+        else:
+            main.log.error( counterName + " counter value is incorrect," +
+                            " expected value: " + str( counterValue )
+                            + " current value: " + str( onosValue ) )
+            correctResults = main.FALSE
+    return consistent and correctResults
diff --git a/TestON/tests/HAkillNodes/dependencies/__init__.py b/TestON/tests/HAkillNodes/dependencies/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HAkillNodes/dependencies/__init__.py
diff --git a/TestON/tests/HAkillNodes/dependencies/obelisk.py b/TestON/tests/HAkillNodes/dependencies/obelisk.py
new file mode 100755
index 0000000..d613806
--- /dev/null
+++ b/TestON/tests/HAkillNodes/dependencies/obelisk.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+from mininet.topo import Topo
+
+class ObeliskTopo( Topo ):
+    def __init__( self ):
+        Topo.__init__( self )
+        topSwitch = self.addSwitch('s1',dpid='1000'.zfill(16))
+        leftTopSwitch = self.addSwitch('s2',dpid='2000'.zfill(16))
+        rightTopSwitch = self.addSwitch('s5',dpid='5000'.zfill(16))
+        leftBotSwitch = self.addSwitch('s3',dpid='3000'.zfill(16))
+        rightBotSwitch = self.addSwitch('s6',dpid='6000'.zfill(16))
+        midBotSwitch = self.addSwitch('s28',dpid='2800'.zfill(16))
+
+        topHost = self.addHost( 'h1' )
+        leftTopHost = self.addHost('h2')
+        rightTopHost = self.addHost('h5')
+        leftBotHost = self.addHost('h3')
+        rightBotHost = self.addHost('h6')
+        midBotHost = self.addHost('h28')
+        self.addLink(topSwitch,topHost)
+        self.addLink(leftTopSwitch,leftTopHost)
+        self.addLink(rightTopSwitch,rightTopHost)
+        self.addLink(leftBotSwitch,leftBotHost)
+        self.addLink(rightBotSwitch,rightBotHost)
+        self.addLink(midBotSwitch,midBotHost)
+        self.addLink(leftTopSwitch,rightTopSwitch)
+        self.addLink(topSwitch,leftTopSwitch)
+        self.addLink(topSwitch,rightTopSwitch)
+        self.addLink(leftTopSwitch,leftBotSwitch)
+        self.addLink(rightTopSwitch,rightBotSwitch)
+        self.addLink(leftBotSwitch,midBotSwitch)
+        self.addLink(midBotSwitch,rightBotSwitch)
+
+        agg1Switch = self.addSwitch('s4',dpid = '3004'.zfill(16))
+        agg2Switch = self.addSwitch('s7',dpid = '6007'.zfill(16))
+        agg1Host = self.addHost('h4')
+        agg2Host = self.addHost('h7')
+        self.addLink(agg1Switch,agg1Host)
+        self.addLink(agg2Switch,agg2Host)
+        self.addLink(agg1Switch, leftBotSwitch)
+        self.addLink(agg2Switch, rightBotSwitch)
+
+        for i in range(10):
+            num = str(i+8)
+            switch = self.addSwitch('s'+num,dpid = ('30'+num.zfill(2)).zfill(16))
+            host = self.addHost('h'+num)
+            self.addLink(switch, host)
+            self.addLink(switch, agg1Switch)
+
+        for i in range(10):
+            num = str(i+18)
+            switch = self.addSwitch('s'+num,dpid = ('60'+num.zfill(2)).zfill(16))
+            host = self.addHost('h'+num)
+            self.addLink(switch, host)
+            self.addLink(switch, agg2Switch)
+
+topos = { 'obelisk': (lambda: ObeliskTopo() ) }
+
+def run():
+    topo = ObeliskTopo()
+    net = Mininet( topo=topo, controller=RemoteController, autoSetMacs=True )
+    net.start()
+    CLI( net )
+    net.stop()
+
+if __name__ == '__main__':
+    setLogLevel( 'info' )
+    run()
diff --git a/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions b/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
new file mode 100755
index 0000000..b73ed4a
--- /dev/null
+++ b/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+'''
+  Generate the partitions json file from the $OC* environment variables
+
+  Usage: onos-gen-partitions [output file]
+  If output file is not provided, the json is written to stdout.
+'''
+
+from os import environ
+from collections import deque, OrderedDict
+import re
+import json
+import sys
+
+convert = lambda text: int(text) if text.isdigit() else text.lower()
+alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
+
+def get_OC_vars():
+  vars = []
+  for var in environ:
+    if re.match(r"OC[0-9]+", var):
+      vars.append(var)
+  return sorted(vars, key=alphanum_key)
+
+def get_nodes(vars, port=9876):
+  node = lambda k: { 'id': k, 'ip': k, 'port': port }
+  return [ node(environ[v]) for v in vars ]
+
+def generate_permutations(nodes, k):
+  l = deque(nodes)
+  perms = []
+  for i in range(1, len(nodes)+1):
+    part = {
+             'name': 'p%d' % i,
+             'members': list(l)[:k]
+           }
+    perms.append(part)
+    l.rotate(-1)
+  return perms
+
+def generate_permutations2(nodes, k):
+  l = deque(nodes)
+  perms = []
+  for i in range(1, (len(nodes) + 1) / 2 + 1):
+    part = {
+             'name': 'p%d' % i,
+             'members': list(l)[:k]
+           }
+    perms.append(part)
+    l.rotate(-2)
+  return perms
+
+if __name__ == '__main__':
+  vars = get_OC_vars()
+  nodes = get_nodes(vars)
+  partitions = generate_permutations2([v.get('id') for v in nodes], 3)
+  data = {
+           'name': 'default',
+           'nodes': nodes,
+           'partitions': partitions
+         }
+  output = json.dumps(data, indent=4)
+
+  if len(sys.argv) == 2:
+    filename = sys.argv[1]
+    with open(filename, 'w') as f:
+      f.write(output)
+  else:
+    print output