Merge "Update HA tests to Atomix 3.x"
diff --git a/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy b/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
index a0905a3..ef20820 100644
--- a/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
+++ b/TestON/JenkinsFile/dependencies/JenkinsCommonFuncs.groovy
@@ -267,7 +267,8 @@
       echo "Job name does not match any test suite name to move log!"
   else
       pwd
-      for i in $OC{1..7}; do onos-fetch-logs $i || echo log does not exist; done
+      for i in $OC{1..7}; do onos-fetch-logs $i || echo log does not exist for onos $i; done
+      for i in $OC{1..7}; do atomix-fetch-logs $i || echo log does not exist for atomix $i; done
   fi
   cd'''
 }
diff --git a/TestON/JenkinsFile/dependencies/TriggerFuncs.groovy b/TestON/JenkinsFile/dependencies/TriggerFuncs.groovy
index eacb1a6..78735f1 100644
--- a/TestON/JenkinsFile/dependencies/TriggerFuncs.groovy
+++ b/TestON/JenkinsFile/dependencies/TriggerFuncs.groovy
@@ -232,15 +232,14 @@
 }
 
 def postSetup( onos_branch, test_branch, onos_tag, isManual ){
-    // setup that will build the onos using buck.
+    // setup that will build ONOS
 
     result = ""
     if ( !isManual ){
         result = '''echo -e "\n##### build ONOS skip unit tests ######"
-        #mvn clean install -DskipTests
-        # Force buck update
-        rm -f ~/onos/bin/buck
-        ~/onos/tools/build/onos-buck build onos
+        cd ~/onos
+        . tools/dev/bash_profile
+        op
         sleep 30
         echo -e "\n##### Stop all running instances of Karaf #####"
         kill $(ps -efw | grep karaf | grep -v grep | awk '{print $2}')
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 06a1a48..dd60fb4 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -545,9 +545,8 @@
                 logStr = "\"Sending CLI command: '" + cmdStr + "'\""
                 self.log( logStr, noExit=noExit )
             self.handle.sendline( cmdStr )
-            i = self.handle.expect( "onos>", timeout )
+            self.handle.expect( "onos>", timeout )
             response = self.handle.before
-            # TODO: do something with i
             main.log.info( "Command '" + str( cmdStr ) + "' sent to "
                            + self.name + "." )
             if debug:
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 08138c2..2593b6e 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -29,7 +29,6 @@
 import os
 import re
 import subprocess
-from requests.models import Response
 from drivers.common.clidriver import CLI
 
 class OnosDriver( CLI ):
@@ -712,8 +711,7 @@
     def getBranchName( self ):
         import re
         try:
-            main.log.info( "self.home = " )
-            main.log.info( self.home )
+            main.log.info( self.name + " home is " + self.home )
             self.handle.sendline( "cd " + self.home )
             self.handle.expect( self.prompt )
             self.handle.sendline( "git name-rev --name-only HEAD" )
@@ -788,7 +786,8 @@
             main.cleanAndExit()
 
     def createCellFile( self, benchIp, fileName, mnIpAddrs,
-                        appString, onosIpAddrs, onosUser="sdn", useSSH=True ):
+                        appString, onosIpAddrs, atomixIps,
+                        onosUser="sdn", useSSH=True ):
         """
         Creates a cell file based on arguments
         Required:
@@ -817,6 +816,8 @@
             cellFile = open( tempDirectory + fileName, 'w+' )
             if isinstance( onosIpAddrs, types.StringType ):
                 onosIpAddrs = [ onosIpAddrs ]
+            if isinstance( atomixIps, types.StringType ):
+                atomixIps = [ atomixIps ]
 
             # App string is hardcoded environment variables
             # That you may wish to use by default on startup.
@@ -831,7 +832,7 @@
             if mnIpAddrs == "":
                 mnString = ""
             onosString = "export OC"
-            tempCount = 1
+            atomixString = "export OCC"
 
             # Create ONOSNIC ip address prefix
             tempOnosIp = str( onosIpAddrs[ 0 ] )
@@ -847,14 +848,21 @@
             # Start writing to file
             cellFile.write( onosNicString + "\n" )
 
+            onosIndex = 1
             for arg in onosIpAddrs:
                 # For each argument in onosIpAddrs, write to file
                 # Output should look like the following:
                 #   export OC1="10.128.20.11"
                 #   export OC2="10.128.20.12"
-                cellFile.write( onosString + str( tempCount ) +
+                cellFile.write( onosString + str( onosIndex ) +
                                 "=\"" + arg + "\"\n" )
-                tempCount = tempCount + 1
+                onosIndex = onosIndex + 1
+
+            atomixIndex = 1
+            for ip in atomixIps:
+                cellFile.write( atomixString + str( atomixIndex ) +
+                                    "=\"" + ip + "\"\n" )
+                atomixIndex += 1
 
             cellFile.write( "export OCI=$OC1\n" )
             if mnString:
@@ -942,7 +950,7 @@
             main.log.info( "Verify cell returned: " + handleBefore +
                            handleAfter )
             return main.TRUE
-        except pexpect.ExceptionPexpect as e:
+        except pexpect.ExceptionPexpect:
             main.log.exception( self.name + ": Pexpect exception found: " )
             main.log.error( self.name + ":    " + self.handle.before )
             main.cleanAndExit()
@@ -982,7 +990,7 @@
             if "value=" + paramValue + "," in self.handle.before:
                 main.log.info( "cfg " + configName + " successfully set to " + configParam )
                 return main.TRUE
-        except pexpect.ExceptionPexpect as e:
+        except pexpect.ExceptionPexpect:
             main.log.exception( self.name + ": Pexpect exception found: " )
             main.log.error( self.name + ":    " + self.handle.before )
             main.cleanAndExit()
@@ -1072,15 +1080,17 @@
                                       pexpect.TIMEOUT ], timeout=180 )
             if i == 0:
                 # can't reach ONOS node
-                main.log.warn( "Network is unreachable" )
+                main.log.warn( self.name + ": Network is unreachable" )
                 self.handle.expect( self.prompt )
                 return main.FALSE
             elif i == 1:
                 # Process started
-                main.log.info(
-                    "Secure SSH performed on " +
-                    node )
+                main.log.info( self.name + ": Secure SSH performed on " + node )
                 return main.TRUE
+            elif i == 2:
+                # timeout
+                main.log.error( self.name + ": Failed to secure ssh on " + node )
+                main.log.debug( self.handle.before )
         except pexpect.EOF:
             main.log.error( self.name + ": EOF exception found" )
             main.log.error( self.name + ":    " + self.handle.before )
@@ -1139,7 +1149,8 @@
                 return main.FALSE
             elif i == 4:
                 # prompt
-                main.log.info( "ONOS was installed on " + node )
+                main.log.info( "ONOS was installed on {} {}.".format(  node,
+                               "but not started" if 'n' in options else "and started" ) )
                 return main.TRUE
             elif i == 5:
                 # timeout
@@ -1429,9 +1440,9 @@
                 # NOTE: since this function won't return until ONOS is ready,
                 #   we will kill it on timeout
                 if i == 1:
-                    main.log.error( "ONOS has not started yet" )
+                    main.log.error( "{}: ONOS {} has not started yet".format( self.name, node ) )
                 elif i == 2:
-                    main.log.error( "Cannot login to ONOS CLI, try using onos-secure-ssh" )
+                    main.log.error( "{}: Cannot login to ONOS CLI {}, try using onos-secure-ssh".format( self.name, node ) )
                 self.handle.send( "\x03" )  # Control-C
                 self.handle.expect( self.prompt )
                 return main.FALSE
@@ -2162,10 +2173,10 @@
             main.cleanAndExit()
         except AssertionError:
             main.log.info( "Settings did not post to ONOS" )
-            main.log.error( varification )
+            main.log.error( verification )
         except Exception:
             main.log.exception( self.name + ": Uncaught exception!" )
-            main.log.error( varification )
+            main.log.error( verification )
             main.cleanAndExit()
 
     def getOnosIps( self ):
@@ -2356,7 +2367,7 @@
         DBString = ""
 
         for item in testData:
-            if isinstance( item, string ):
+            if isinstance( item, str ):
                 item = "'" + item + "'"
             if testData.index( item ) < len( testData - 1 ):
                 item += ","
@@ -2469,10 +2480,12 @@
         import time
 
         self.createCellFile( self.ip_address,
-                                       "temp",
-                                       self.ip_address,
-                                       "drivers",
-                                       nodeList, onosUser )
+                             "temp",
+                             self.ip_address,
+                             "drivers",
+                             nodeList,
+                             nodeList,
+                             onosUser=onosUser )
 
         main.log.info( self.name + ": Apply cell to environment" )
         cellResult = self.setCell( "temp" )
@@ -2801,21 +2814,21 @@
                                       pexpect.TIMEOUT ], timeout=180 )
             if i == 0:
                 # can't reach ONOS node
-                main.log.warn( "Network is unreachable" )
+                main.log.warn( self.name + ": Network is unreachable" )
                 self.handle.expect( self.prompt )
                 return main.FALSE
             elif i == 1:
                 # same bits are already on Atomix node
-                main.log.info( "Atomix is already installed on " + node )
+                main.log.info( self.name + ": Atomix is already installed on " + node )
                 self.handle.expect( self.prompt )
                 return main.TRUE
             elif i == 2 or i == 3:
-                main.log.info( "Atomix was installed on " + node )
+                main.log.info( self.name + ": Atomix was installed on " + node )
                 self.handle.expect( self.prompt )
                 return main.TRUE
             elif i == 4:
                 # timeout
-                main.log.info( "Installation of Atomix on " + node + " timed out" )
+                main.log.info( self.name + ": Installation of Atomix on " + node + " timed out" )
                 self.handle.expect( self.prompt )
                 main.log.warn( self.handle.before )
                 self.handle.send( "\x03" )  # Control-C
diff --git a/TestON/tests/FUNC/FUNCbgpls/FUNCbgpls.py b/TestON/tests/FUNC/FUNCbgpls/FUNCbgpls.py
index 8275033..91630ab 100644
--- a/TestON/tests/FUNC/FUNCbgpls/FUNCbgpls.py
+++ b/TestON/tests/FUNC/FUNCbgpls/FUNCbgpls.py
@@ -77,7 +77,7 @@
             main.testSetUp.envSetupException( e )
 
         cliResults = main.testSetUp.ONOSSetUp( main.Cluster, cellName=main.cellName,
-                                               mininetIp=main.scapy_ip, removeLog=True )
+                                               mininetIp=main.scapy_ip )
         main.step( "App Ids check" )
         appCheck = main.Cluster.active( 0 ).CLI.appToIDCheck()
 
diff --git a/TestON/tests/FUNC/FUNCformCluster/FUNCformCluster.py b/TestON/tests/FUNC/FUNCformCluster/FUNCformCluster.py
index 55b6e41..2a1ecb4 100644
--- a/TestON/tests/FUNC/FUNCformCluster/FUNCformCluster.py
+++ b/TestON/tests/FUNC/FUNCformCluster/FUNCformCluster.py
@@ -86,6 +86,7 @@
                                     main.Mininet1.ip_address,
                                     main.apps,
                                     cluster.ip_address,
+                                    cluster.ip_address,
                                     main.ONOScell.karafUser,
                                     True ] )
             threads.append( t )
diff --git a/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.py b/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.py
index 4570863..3aa7011 100644
--- a/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.py
+++ b/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.py
@@ -79,7 +79,7 @@
         main.testSetUp.evnSetupConclusion( stepResult )
 
         cliResults = main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
-                                               mininetIp=main.OVSDB1, removeLog=True )
+                                               mininetIp=main.OVSDB1 )
 
         if cliResults == main.FALSE:
             main.log.error( "Failed to start ONOS, stopping test" )
diff --git a/TestON/tests/FUNC/FUNCvirNetNB/FUNCvirNetNB.py b/TestON/tests/FUNC/FUNCvirNetNB/FUNCvirNetNB.py
index 2cb3832..d7872fd 100644
--- a/TestON/tests/FUNC/FUNCvirNetNB/FUNCvirNetNB.py
+++ b/TestON/tests/FUNC/FUNCvirNetNB/FUNCvirNetNB.py
@@ -87,7 +87,7 @@
         main.maxNodes = 1
 
         cliResults = main.testSetUp.ONOSSetUp( main.Cluster,
-                                               cellName=cellName, removeLog=True )
+                                               cellName=cellName )
         if cliResults == main.FALSE:
             main.log.error( "Failed to start ONOS, stopping test" )
             main.cleanAndExit()
diff --git a/TestON/tests/HA/HAbackupRecover/HAbackupRecover.py b/TestON/tests/HA/HAbackupRecover/HAbackupRecover.py
index e907b66..7010f93 100644
--- a/TestON/tests/HA/HAbackupRecover/HAbackupRecover.py
+++ b/TestON/tests/HA/HAbackupRecover/HAbackupRecover.py
@@ -97,8 +97,7 @@
         except (KeyError, IndexError):
             applyFuncs = main.HA.startingMininet
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
-                                  extraApply=applyFuncs )
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, extraApply=applyFuncs )
 
         main.HA.initialSetUp()
 
diff --git a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
index 0166ff5..14a0898 100644
--- a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
@@ -97,8 +97,7 @@
         except (KeyError, IndexError):
             applyFuncs = main.HA.startingMininet
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
-                                  extraApply=applyFuncs )
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, extraApply=applyFuncs )
 
         main.HA.initialSetUp()
 
diff --git a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
index 23d30b5..85ad546 100644
--- a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
+++ b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
@@ -103,7 +103,7 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
                                   extraClean=main.HA.cleanUpGenPartition,
diff --git a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
index 886c672..92815f6 100644
--- a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
+++ b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
@@ -102,7 +102,7 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
                                   extraClean=main.HA.cleanUpGenPartition,
diff --git a/TestON/tests/HA/HAkillNodes/HAkillNodes.py b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
index 44a4b71..45852af 100644
--- a/TestON/tests/HA/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
@@ -102,7 +102,7 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
                                   extraClean=main.HA.cleanUpGenPartition,
diff --git a/TestON/tests/HA/HApowerFailure/HApowerFailure.py b/TestON/tests/HA/HApowerFailure/HApowerFailure.py
index dd40e9c..e8fbe61 100644
--- a/TestON/tests/HA/HApowerFailure/HApowerFailure.py
+++ b/TestON/tests/HA/HApowerFailure/HApowerFailure.py
@@ -102,7 +102,7 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
                                   extraClean=main.HA.cleanUpGenPartition,
diff --git a/TestON/tests/HA/HAsanity/HAsanity.py b/TestON/tests/HA/HAsanity/HAsanity.py
index f891a85..b6d00c0 100644
--- a/TestON/tests/HA/HAsanity/HAsanity.py
+++ b/TestON/tests/HA/HAsanity/HAsanity.py
@@ -96,7 +96,7 @@
         except (KeyError, IndexError):
             applyFuncs = main.HA.startingMininet
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs, stopAtomix=True,
                                   includeCaseDesc=False )
         main.HA.initialSetUp()
diff --git a/TestON/tests/HA/HAscaling/HAscaling.params b/TestON/tests/HA/HAscaling/HAscaling.params
index 8478214..9f3ddfb 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.params
+++ b/TestON/tests/HA/HAscaling/HAscaling.params
@@ -20,18 +20,14 @@
     #CASE15:  Check that Leadership Election is still functional
     #CASE16:  Install Distributed Primitives app
     #CASE17:  Check for basic functionality with distributed primitives
-    <testcases>1,2,8,21,3,8,4,5,14,16,17,[6,8,7,4,15,17]*17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+    <testcases>1,2,8,21,3,8,4,5,14,16,17,[6,8,7,4,15,17]*6,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
 
     <GRAPH>
         <nodeCluster>VM</nodeCluster>
         <builds>20</builds>
     </GRAPH>
 
-    <scaling>1,3b,3,3b,5b,5,5b,7b,7,7b,5b,5,5b,3b,3,3b,1</scaling>
-    <server>
-        <port>8000</port>
-        <interface></interface>
-    </server>
+    <scaling>1,3,5,7,5,3,1</scaling>
     <apps></apps>
     <ONOS_Configuration>
         <org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
diff --git a/TestON/tests/HA/HAscaling/HAscaling.py b/TestON/tests/HA/HAscaling/HAscaling.py
index 4e18b30..6929ae7 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.py
+++ b/TestON/tests/HA/HAscaling/HAscaling.py
@@ -65,6 +65,7 @@
         start cli sessions
         start tcpdump
         """
+        import re
         main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
                          "initialization" )
         # set global variables
@@ -82,8 +83,6 @@
         try:
             from tests.HA.dependencies.HA import HA
             main.HA = HA()
-            from tests.HA.HAswapNodes.dependencies.Server import Server
-            main.Server = Server()
             # load some variables from the params file
             cellName = main.params[ 'ENV' ][ 'cellName' ]
             main.apps = main.params[ 'ENV' ][ 'appString' ]
@@ -92,11 +91,14 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.setServerForCluster,
-                       main.HA.scalingMetadata,
-                       main.HA.copyBackupConfig,
-                       main.HA.setMetadataUrl ]
-        applyArgs = [ None, None, None, None ]
+        main.scaling = main.params[ 'scaling' ].split( "," )
+        main.log.debug( main.scaling )
+        scale = main.scaling.pop( 0 )
+        main.log.debug( scale )
+        main.Cluster.setRunningNode( int( re.search( "\d+", scale ).group( 0 ) ) )
+
+        applyFuncs = []
+        applyArgs = []
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
@@ -107,13 +109,13 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
-                                  extraClean=main.HA.cleanUpOnosService,
                                   installMax=True,
+                                  atomixClusterSize=3,
                                   includeCaseDesc=False )
-        main.HA.initialSetUp( serviceClean=True )
+        main.HA.initialSetUp()
 
         main.step( 'Set logging levels' )
         logging = True
@@ -203,37 +205,39 @@
             main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
 
         """
-        pop # of nodes from a list, might look like 1,3b,3,5b,5,7b,7,7b,5,5b,3...
-        modify cluster.json file appropriately
+        pop # of nodes from a list, might look like 1,3,5,7,5,3...
         install/deactivate node as needed
         """
         try:
-            prevNodes = main.Cluster.active()
+            prevNodes = main.Cluster.getRunningPos()
+            prevSize = main.Cluster.numCtrls
             scale = main.scaling.pop( 0 )
-            if "b" in scale:
-                equal = True
-            else:
-                equal = False
             main.Cluster.setRunningNode( int( re.search( "\d+", scale ).group( 0 ) ) )
-            main.step( "Scaling to {} nodes; Equal partitions: {}".format( main.Cluster.numCtrls, equal ) )
-            genResult = main.Server.generateFile( main.Cluster.numCtrls, equal=equal )
-            utilities.assert_equals( expect=main.TRUE, actual=genResult,
-                                     onpass="New cluster metadata file generated",
-                                     onfail="Failled to generate new metadata file" )
-            time.sleep( 5 )  # Give time for nodes to read new file
-        except IndexError:
+            main.step( "Scaling from {} to {} nodes".format(
+                prevSize, main.Cluster.numCtrls ) )
+        except IndexError as e:
+            main.log.debug( e )
             main.cleanAndExit()
 
         activeNodes = range( 0, main.Cluster.numCtrls )
         newNodes = [ x for x in activeNodes if x not in prevNodes ]
+        deadNodes = [ x for x in prevNodes if x not in activeNodes ]
         main.Cluster.clearActive()
         main.step( "Start new nodes" )  # OR stop old nodes?
         started = main.TRUE
+        stopped = main.TRUE
         for i in newNodes:
-            started = main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) and main.TRUE
+            main.log.debug( "Starting " + str( main.Cluster.runningNodes[ i ].ipAddress ) )
+            started = main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) and started
         utilities.assert_equals( expect=main.TRUE, actual=started,
                                  onpass="ONOS started",
                                  onfail="ONOS start NOT successful" )
+        for i in deadNodes:
+            main.log.debug( "Stopping " + str( main.Cluster.controllers[ i ].ipAddress ) )
+            stopped = main.ONOSbench.onosStop( main.Cluster.controllers[ i ].ipAddress ) and stopped
+        utilities.assert_equals( expect=main.TRUE, actual=stopped,
+                                 onpass="ONOS stopped",
+                                 onfail="ONOS stop NOT successful" )
 
         main.testSetUp.setupSsh( main.Cluster )
 
@@ -244,7 +248,7 @@
         main.step( "Checking ONOS nodes" )
         nodeResults = utilities.retry( main.Cluster.nodesCheck,
                                        False,
-                                       attempts=5 )
+                                       attempts=90 )
         utilities.assert_equals( expect=True, actual=nodeResults,
                                  onpass="Nodes check successful",
                                  onfail="Nodes check NOT successful" )
@@ -275,8 +279,6 @@
                                  onpass="Reran for election",
                                  onfail="Failed to rerun for election" )
 
-        # TODO: Make this configurable
-        time.sleep( 60 )
         main.HA.commonChecks()
 
     def CASE7( self, main ):
@@ -356,13 +358,6 @@
         """
         main.HA.cleanUp( main )
 
-        main.step( "Stopping webserver" )
-        status = main.Server.stop()
-        utilities.assert_equals( expect=main.TRUE, actual=status,
-                                 onpass="Stop Server",
-                                 onfail="Failled to stop SimpleHTTPServer" )
-        del main.Server
-
     def CASE14( self, main ):
         """
         Start election app on all onos nodes
diff --git a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.params b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.params
index bc22bb4..9f905db 100644
--- a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.params
+++ b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.params
@@ -40,7 +40,7 @@
         <org.onosproject.events>TRACE</org.onosproject.events>
     </ONOS_Logging>
     <ENV>
-        <cellName>HA</cellName>
+        <cellName>SingleHA</cellName>
         <appString>events,drivers,openflow,proxyarp,mobility</appString>
     </ENV>
     <GIT>
diff --git a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index 4f1ec82..4fbdb01 100644
--- a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -91,8 +91,10 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
+        cellApps = str( main.params["ENV"]["appString"] )
+        cellNAme = str( main.params["ENV"]["appString"] )
         applyFuncs = [ main.testSetUp.createApplyCell ]
-        applyArgs = [ [ main.Cluster, True, "SingleHA", "", "", True, main.Cluster.runningNodes[ 0 ].ipAddress ] ]
+        applyArgs = [ [ main.Cluster, True, cellName , cellApps, "", True, main.Cluster.runningNodes[ 0 ].ipAddress ] ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
@@ -105,7 +107,7 @@
 
         main.Cluster.setRunningNode( int( main.params[ 'num_controllers' ] ) )
         ip = main.Cluster.getIps( allNode=True )
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName="SingleHA", removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName="SingleHA",
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
                                   includeCaseDesc=False )
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.py b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
index 62b8ee5..e112f9f 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
@@ -100,7 +100,7 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
                                   extraClean=main.HA.cleanUpGenPartition,
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.params b/TestON/tests/HA/HAswapNodes/HAswapNodes.params
index 8ed012a..e592770 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.params
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.params
@@ -27,10 +27,6 @@
         <builds>20</builds>
     </GRAPH>
 
-    <server>
-        <port>8000</port>
-        <interface></interface>
-    </server>
     <apps></apps>
     <ONOS_Configuration>
         <org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.py b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
index bf1a589..52da8ed 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.py
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
@@ -81,8 +81,6 @@
         try:
             from tests.HA.dependencies.HA import HA
             main.HA = HA()
-            from tests.HA.HAswapNodes.dependencies.Server import Server
-            main.Server = Server()
             # load some variables from the params file
             cellName = main.params[ 'ENV' ][ 'cellName' ]
             main.apps = main.params[ 'ENV' ][ 'appString' ]
@@ -91,11 +89,8 @@
             main.testSetUp.envSetupException( e )
         main.testSetUp.evnSetupConclusion( stepResult )
 
-        applyFuncs = [ main.HA.setServerForCluster,
-                       main.HA.swapNodeMetadata,
-                       main.HA.copyBackupConfig,
-                       main.HA.setMetadataUrl ]
-        applyArgs = [ None, None, None, None ]
+        applyFuncs = [ main.HA.swapNodeMetadata ]
+        applyArgs = [ None ]
         try:
             if main.params[ 'topology' ][ 'topoFile' ]:
                 main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
@@ -106,13 +101,12 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
-                                  extraClean=main.HA.cleanUpOnosService,
                                   installMax=True,
                                   includeCaseDesc=False )
-        main.HA.initialSetUp( serviceClean=True )
+        main.HA.initialSetUp()
 
         main.step( 'Set logging levels' )
         logging = True
@@ -179,7 +173,6 @@
         """
         The Scaling case.
         """
-        import time
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         try:
@@ -201,8 +194,6 @@
             main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
 
         activeNodes = main.Cluster.getRunningPos()
-        # Todo : this could be wrong. need to double check.
-        main.step( "Generate new metadata file" )
         old = [ activeNodes[ 1 ], activeNodes[ -2 ] ]
         new = range( main.Cluster.maxCtrls )[ -2: ]
         assert len( old ) == len( new ), "Length of nodes to swap don't match"
@@ -217,20 +208,22 @@
             activeNodes.remove( x )
             activeNodes.append( y )
 
-        genResult = main.Server.generateFile( main.Cluster.numCtrls )
-        utilities.assert_equals( expect=main.TRUE, actual=genResult,
-                                 onpass="New cluster metadata file generated",
-                                 onfail="Failled to generate new metadata file" )
-        time.sleep( 5 )  # Give time for nodes to read new file
         main.Cluster.clearActive()
         # Note : done up to this point.
         main.step( "Start new nodes" )  # OR stop old nodes?
         started = main.TRUE
+        stopped = main.TRUE
         for i in new:
-            started = main.ONOSbench.onosStart( main.Cluster.controllers[ i ].ipAddress ) and main.TRUE
+            started = main.ONOSbench.onosStart( main.Cluster.controllers[ i ].ipAddress ) and started
         utilities.assert_equals( expect=main.TRUE, actual=started,
                                  onpass="ONOS started",
                                  onfail="ONOS start NOT successful" )
+        for i in old:
+            main.log.debug( "Stopping " + str( main.Cluster.controllers[ i ].ipAddress ) )
+            stopped = main.ONOSbench.onosStop( main.Cluster.controllers[ i ].ipAddress ) and stopped
+        utilities.assert_equals( expect=main.TRUE, actual=stopped,
+                                 onpass="ONOS stopped",
+                                 onfail="ONOS stop NOT successful" )
 
         main.Cluster.setRunningNode( activeNodes )
 
@@ -242,16 +235,16 @@
         main.step( "Checking ONOS nodes" )
         nodeResults = utilities.retry( main.Cluster.nodesCheck,
                                        False,
-                                       attempts=5 )
+                                       attempts=15 )
         utilities.assert_equals( expect=True, actual=nodeResults,
                                  onpass="Nodes check successful",
                                  onfail="Nodes check NOT successful" )
 
         ready = utilities.retry( main.Cluster.command,
-                                  False,
-                                  kwargs={ "function": "summary", "contentCheck": True },
-                                  sleep=30,
-                                  attempts=10 )
+                                 False,
+                                 kwargs={ "function": "summary", "contentCheck": True },
+                                 sleep=30,
+                                 attempts=10 )
         utilities.assert_equals( expect=True, actual=ready,
                                  onpass="ONOS summary command succeded",
                                  onfail="ONOS summary command failed" )
@@ -348,13 +341,6 @@
         """
         main.HA.cleanUp( main )
 
-        main.step( "Stopping webserver" )
-        status = main.Server.stop()
-        utilities.assert_equals( expect=main.TRUE, actual=status,
-                                 onpass="Stop Server",
-                                 onfail="Failled to stop SimpleHTTPServer" )
-        del main.Server
-
     def CASE14( self, main ):
         """
         Start election app on all onos nodes
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.py b/TestON/tests/HA/HAupgrade/HAupgrade.py
index 3eb6c94..9f1d307 100644
--- a/TestON/tests/HA/HAupgrade/HAupgrade.py
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.py
@@ -104,7 +104,7 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
                                   extraClean=main.HA.cleanUpGenPartition,
diff --git a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
index 8b81f59..3101324 100644
--- a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
+++ b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
@@ -104,7 +104,7 @@
                 applyFuncs.append( main.HA.startingMininet )
                 applyArgs.append( None )
 
-        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+        main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName,
                                   extraApply=applyFuncs,
                                   applyArgs=applyArgs,
                                   extraClean=main.HA.cleanUpGenPartition,
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index 8581248..a4ff617 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -71,47 +71,11 @@
                                  onpass="Mininet Started",
                                  onfail="Error starting Mininet" )
 
-    def scalingMetadata( self ):
-        main.step( "Generate initial metadata file" )
-        main.scaling = main.params[ 'scaling' ].split( "," )
-        main.log.debug( main.scaling )
-        scale = main.scaling.pop( 0 )
-        main.log.debug( scale )
-        if "b" in scale:
-            equal = True
-        else:
-            equal = False
-        main.log.debug( equal )
-        main.Cluster.setRunningNode( int( re.search( "\d+", scale ).group( 0 ) ) )
-        genResult = main.Server.generateFile( main.Cluster.numCtrls, equal=equal )
-        utilities.assert_equals( expect=main.TRUE, actual=genResult,
-                                 onpass="New cluster metadata file generated",
-                                 onfail="Failled to generate new metadata file" )
-
     def swapNodeMetadata( self ):
-        main.step( "Generate initial metadata file" )
         if main.Cluster.numCtrls >= 5:
             main.Cluster.setRunningNode( main.Cluster.numCtrls - 2 )
         else:
             main.log.error( "Not enough ONOS nodes to run this test. Requires 5 or more" )
-        genResult = main.Server.generateFile( main.Cluster.numCtrls )
-        utilities.assert_equals( expect=main.TRUE, actual=genResult,
-                                 onpass="New cluster metadata file generated",
-                                 onfail="Failled to generate new metadata file" )
-
-    def setServerForCluster( self ):
-        import os
-        main.step( "Setup server for cluster metadata file" )
-        main.serverPort = main.params[ 'server' ][ 'port' ]
-        rootDir = os.path.dirname( main.testFile ) + "/dependencies"
-        main.log.debug( "Root dir: {}".format( rootDir ) )
-        status = main.Server.start( main.ONOSbench,
-                                    rootDir,
-                                    port=main.serverPort,
-                                    logDir=main.logdir + "/server.log" )
-        utilities.assert_equals( expect=main.TRUE, actual=status,
-                                 onpass="Server started",
-                                 onfail="Failled to start SimpleHTTPServer" )
 
     def copyBackupConfig( self ):
         main.step( "Copying backup config files" )
@@ -181,7 +145,8 @@
             onosCounters = []
             for i in range( len( onosCountersRaw ) ):
                 try:
-                    onosCounters.append( json.loads( onosCountersRaw[ i ] ) )
+                    value = json.loads( onosCountersRaw[ i ] )
+                    onosCounters.append( value )
                 except ( ValueError, TypeError ):
                     main.log.error( "Could not parse counters response from " +
                                     str( main.Cluster.active( i ) ) )
@@ -2519,6 +2484,8 @@
         main.step( "Checking raft log size" )
         # TODO: this is a flaky check, but the intent is to make sure the raft logs
         #       get compacted periodically
+
+        # FIXME: We need to look at the raft servers, which might not be on the ONOS machine
         logCheck = main.Cluster.checkPartitionSize()
         utilities.assert_equals( expect=True, actual=logCheck,
                                  onpass="Raft log size is not too big",
@@ -2860,7 +2827,8 @@
 
         main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
                                        main.Mininet1.ip_address,
-                                       cellAppString, ipList, main.ONOScli1.karafUser )
+                                       cellAppString, ipList, ipList,
+                                       main.ONOScli1.karafUser )
         main.step( "Applying cell variable to environment" )
         cellResult = main.ONOSbench.setCell( cellName )
         verifyResult = main.ONOSbench.verifyCell()
@@ -2879,7 +2847,7 @@
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         main.case( "Running ONOS Constant State Tests" )
 
-        OnosAfterWhich = [ "failure", "scaliing" ]
+        OnosAfterWhich = [ "failure", "scaling" ]
 
         # Assert that each device has a master
         self.checkRoleNotNull()
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index 29966d6..6d82489 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -26,10 +26,16 @@
 
     def __repr__( self ):
         controllers = []
-        runningNodes = []
+        runningNodes = self.getRunningNodes()
+        atomixNodes = []
         for ctrl in self.controllers:
-            controllers.append( str( ctrl ) )
-        return "%s[%s]" % ( self.name, ", ".join( controllers ) )
+            controllers.append( "{%s:%s, %s - %s}" % ( ctrl.name,
+                                                       ctrl.ipAddress,
+                                                       "Configured" if ctrl in runningNodes else "Not Configured",
+                                                       "Active" if ctrl.active else "Inactive" ) )
+        for node in self.atomixNodes:
+            atomixNodes.append( "{%s:%s}" % ( node.name, node.ipAddress ) )
+        return "%s[%s; Atomix Nodes:%s]" % ( self.name, ", ".join( controllers ), ", ".join( atomixNodes ) )
 
     def __init__( self, ctrlList=[], name="Cluster" ):
         """
@@ -45,8 +51,33 @@
         self.numCtrls = len( self.runningNodes )
         self.maxCtrls = len( self.controllers )
         self.name = str( name )
+        self.atomixNodes = ctrlList
         self.iterator = iter( self.active() )
 
+    def fromNode( self, ctrlList ):
+        """
+        Helper function to get a specific list of controllers
+        Required Arguments:
+        * ctrlList - The list of controllers to return. This can be either an index or a keyword
+            Index | Keyword   | List returned
+            0     | "all"     | self.controllers
+            1     | "running" | self.runningNodes
+            2     | "active   | self.active()
+
+        Throws a ValueError exception if ctrlList value is not recognized
+        """
+        # TODO: Add Atomix Nodes?
+        if isinstance( ctrlList, str ):
+            ctrlList = ctrlList.lower()
+        if ctrlList == 0 or ctrlList == "all":
+            return self.controllers
+        elif ctrlList == 1 or ctrlList == "running":
+            return self.runningNodes
+        elif ctrlList == 2 or ctrlList == "active":
+            return self.active()
+        else:
+            raise ValueError( "Unknown argument: {}".format( ctrlList ) )
+
     def getIps( self, activeOnly=False, allNode=False ):
         """
         Description:
@@ -108,15 +139,58 @@
             self.runningNodes.append( self.controllers[ i ] )
         self.numCtrls = len( numCtrls ) if isinstance( numCtrls, list ) else numCtrls
 
+    def setAtomixNodes( self, nodes ):
+        """
+        Description:
+            Sets the list of Atomix nodes for the cluster
+            If nodes is a list, it will add the nodes of the list.
+            If nodes is an int, the function will set the Atomix nodes
+            to be the first n in the list of controllers.
+        Required:
+            * nodes - number of nodes to be set, or a list of nodes to set
+        Returns:
+        """
+        self.atomixNodes = []
+        for i in nodes if isinstance( nodes, list ) else range( nodes ):
+            self.atomixNodes.append( self.controllers[ i ] )
+
+    def getControllers( self, node=None ):
+        """
+        Description:
+            Get the list of all controllers in a cluster or a controller at an index in the list
+        Optional Arguments:
+            * node - position of the node to get from the list of controllers.
+        Returns:
+            Return a list of controllers in the cluster if node is None
+            if not, it will return the controller at the given index.
+        """
+        result = self.controllers
+        return result if node is None else result[ node % len( result ) ]
+
+    def getRunningNodes( self, node=None ):
+        """
+        Description:
+            Get the list of all controllers in a cluster that should be running or
+            a controller at an index in the list
+        Optional Arguments:
+            * node - position of the node to get from the list of controllers.
+        Returns:
+            Return a list of controllers in the cluster if node is None
+            if not, it will return the controller at the given index.
+        """
+        result = self.runningNodes
+        return result if node is None else result[ node % len( result ) ]
+
     def active( self, node=None ):
         """
         Description:
-            get the list/controller of the active controller.
-        Required:
-            * node - position of the node to get from the active controller.
+            Get the list of all active controllers in a cluster or
+            a controller at an index in the list
+        Optional Arguments:
+            * node - position of the node to get from the list of controllers.
         Returns:
-            Return a list of active controllers in the cluster if node is None
-            if not, it will return the nth controller.
+            Return a list of controllers in the cluster if node is None
+            if not, it will return the controller at the given index.
         """
         result = [ ctrl for ctrl in self.runningNodes
                       if ctrl.active ]
@@ -149,7 +223,8 @@
         """
         self.iterator = iter( self.active() )
 
-    def createCell( self, cellName, cellApps, mininetIp, useSSH, ips, installMax=False ):
+    def createCell( self, cellName, cellApps, mininetIp, useSSH, onosIps,
+                    atomixIps, installMax=False ):
         """
         Description:
             create a new cell
@@ -158,7 +233,9 @@
             * cellApps - The ONOS apps string.
             * mininetIp - Mininet IP address.
             * useSSH - True for using ssh when creating a cell
-            * ips - ip( s ) of the node( s ).
+            * onosIps - ip( s ) of the ONOS node( s ).
+            * atomixIps - ip( s ) of the Atomix node( s ).
+
         Returns:
         """
         self.command( "createCellFile",
@@ -166,11 +243,12 @@
                              cellName,
                              mininetIp,
                              cellApps,
-                             ips,
+                             onosIps,
+                             atomixIps,
                              main.ONOScell.karafUser,
                              useSSH ],
                       specificDriver=1,
-                      getFrom=0 if installMax else 1 )
+                      getFrom="all" if installMax else "running" )
 
     def uninstallAtomix( self, uninstallMax ):
         """
@@ -186,7 +264,7 @@
         uninstallResult = self.command( "atomixUninstall",
                                         kwargs={ "nodeIp": "ipAddress" },
                                         specificDriver=1,
-                                        getFrom=0 if uninstallMax else 1,
+                                        getFrom="all" if uninstallMax else "running",
                                         funcFromCtrl=True )
         for uninstallR in uninstallResult:
             result = result and uninstallR
@@ -206,7 +284,7 @@
         uninstallResult = self.command( "onosUninstall",
                                         kwargs={ "nodeIp": "ipAddress" },
                                         specificDriver=1,
-                                        getFrom=0 if uninstallMax else 1,
+                                        getFrom="all" if uninstallMax else "running",
                                         funcFromCtrl=True )
         for uninstallR in uninstallResult:
             result = result and uninstallR
@@ -225,13 +303,15 @@
         setCellResult = self.command( "setCell",
                                       args=[ cellName ],
                                       specificDriver=1,
-                                      getFrom=0 if installMax else 1 )
+                                      getFrom="all" )
+        benchCellResult = main.ONOSbench.setCell( cellName )
         verifyResult = self.command( "verifyCell",
                                      specificDriver=1,
-                                     getFrom=0 if installMax else 1 )
+                                     getFrom="all" )
         result = main.TRUE
         for i in range( len( setCellResult ) ):
             result = result and setCellResult[ i ] and verifyResult[ i ]
+        result = result and benchCellResult
         return result
 
     def checkService( self ):
@@ -243,16 +323,16 @@
         Returns:
             Returns main.TRUE if it successfully checked
         """
-        stopResult = main.TRUE
-        startResult = main.TRUE
+        getFrom = "running"
         onosIsUp = main.TRUE
         onosUp = self.command( "isup",
                                  args=[ "ipAddress" ],
                                  specificDriver=1,
-                                 getFrom=1,
+                                 getFrom=getFrom,
                                  funcFromCtrl=True )
+        ctrlList = self.fromNode( getFrom )
         for i in range( len( onosUp ) ):
-            ctrl = self.controllers[ i ]
+            ctrl = ctrlList[ i ]
             onosIsUp = onosIsUp and onosUp[ i ]
             if onosUp[ i ] == main.TRUE:
                 main.log.info( ctrl.name + " is up and ready" )
@@ -277,11 +357,10 @@
         killResult = self.command( "atomixKill",
                                    args=[ "ipAddress" ],
                                    specificDriver=1,
-                                   getFrom=0 if killMax else 1,
+                                   getFrom="all" if killMax else "running",
                                    funcFromCtrl=True )
         for i in range( len( killResult ) ):
             result = result and killResult[ i ]
-            self.controllers[ i ].active = False
         return result
 
     def killOnos( self, killMax, stopOnos ):
@@ -297,15 +376,17 @@
         Returns:
             Returns main.TRUE if successfully killing it.
         """
+        getFrom = "all" if killMax else "running"
         result = main.TRUE
         killResult = self.command( "onosKill",
                                    args=[ "ipAddress" ],
                                    specificDriver=1,
-                                   getFrom=0 if killMax else 1,
+                                   getFrom=getFrom,
                                    funcFromCtrl=True )
+        ctrlList = self.fromNode( getFrom )
         for i in range( len( killResult ) ):
             result = result and killResult[ i ]
-            self.controllers[ i ].active = False
+            ctrlList[ i ].active = False
         return result
 
     def ssh( self ):
@@ -321,30 +402,25 @@
         sshResult = self.command( "onosSecureSSH",
                                    kwargs={ "node": "ipAddress" },
                                    specificDriver=1,
-                                   getFrom=1,
+                                   getFrom="running",
                                    funcFromCtrl=True )
         for sshR in sshResult:
             result = result and sshR
         return result
 
-    def installAtomix( self, installMax=True, installParallel=True ):
+    def installAtomix( self, installParallel=True ):
         """
         Description:
-            Installing onos.
+            Installing Atomix.
         Required:
-            * installMax - True for installing max number of nodes
-            False for installing current running nodes only.
         Returns:
             Returns main.TRUE if it successfully installed
         """
         result = main.TRUE
         threads = []
         i = 0
-        for ctrl in self.controllers if installMax else self.runningNodes:
-            options = ""
-            if installMax and i >= self.numCtrls:
-                # TODO: is installMax supported here?
-                pass
+        for ctrl in self.atomixNodes:
+            options = "-f"
             if installParallel:
                 t = main.Thread( target=ctrl.Bench.atomixInstall,
                                  name="atomix-install-" + ctrl.name,
@@ -404,15 +480,17 @@
         Returns:
             Returns main.TRUE if it successfully started.
         """
+        getFrom = "running"
         result = main.TRUE
         cliResults = self.command( "startOnosCli",
                                    args=[ "ipAddress" ],
                                    specificDriver=2,
-                                   getFrom=1,
+                                   getFrom=getFrom,
                                    funcFromCtrl=True )
+        ctrlList = self.fromNode( getFrom )
         for i in range( len( cliResults ) ):
             result = result and cliResults[ i ]
-            self.controllers[ i ].active = True
+            ctrlList[ i ].active = True
         return result
 
     def nodesCheck( self ):
@@ -437,6 +515,8 @@
                 activeIps.sort()
                 if ips == activeIps:
                     currentResult = True
+                else:
+                    main.log.error( "{} != {}".format( ips, activeIps ) )
             except ( ValueError, TypeError ):
                 main.log.error( "Error parsing nodes output" )
                 main.log.warn( repr( i ) )
@@ -526,7 +606,7 @@
         return all( resultOne == result for result in results )
 
     def command( self, function, args=(), kwargs={}, returnBool=False,
-                 specificDriver=0, contentCheck=False, getFrom=2,
+                 specificDriver=0, contentCheck=False, getFrom="active",
                  funcFromCtrl=False ):
         """
         Description:
@@ -546,9 +626,9 @@
             * contentCheck - If this is True, it will check if the result has some
             contents.
             * getFrom - from which nodes
-                2 - active nodes
-                1 - current running nodes
-                0 - all nodes
+                2 or "active" - active nodes
+                1 or "running" - current running nodes
+                0 or "all" - all nodes
             * funcFromCtrl - specific function of the args/kwargs
                  from each controller from the list of the controllers
         Returns:
@@ -558,9 +638,8 @@
         """
         threads = []
         drivers = [ None, "Bench", "CLI", "REST" ]
-        fromNode = [ self.controllers, self.runningNodes, self.active() ]
         results = []
-        for ctrl in fromNode[ getFrom ]:
+        for ctrl in self.fromNode( getFrom ):
             try:
                 funcArgs = []
                 funcKwargs = {}
@@ -587,9 +666,9 @@
             t.join()
             results.append( t.result )
         if returnBool:
-            return self.allTrueResultCheck( results, fromNode[ getFrom ] )
+            return self.allTrueResultCheck( results, self.fromNode( getFrom ) )
         elif contentCheck:
-            return self.notEmptyResultCheck( results, fromNode[ getFrom ] )
+            return self.notEmptyResultCheck( results, self.fromNode( getFrom ) )
         return results
 
     def checkPartitionSize( self, segmentSize='64', units='M', multiplier='3' ):
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index 03328e7..38ba838 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -169,7 +169,7 @@
     def setNumCtrls( self, hasMultiNodeRounds ):
         """
         Description:
-            Set new number of controls if it uses different number of nodes.
+            Set new number of controllers if it uses different number of nodes.
             different number of nodes should be pre-defined in main.scale.
         Required:
             * hasMultiNodeRouds - if the test is testing different number of nodes.
@@ -180,7 +180,7 @@
             except Exception:
                 main.cycle = 0
             main.cycle += 1
-            # main.scale[ 0 ] determines the current number of ONOS controller
+            # main.scale[ 0 ] determines the current number of ONOS controllers
             main.Cluster.setRunningNode( int( main.scale.pop( 0 ) ) )
 
     def killingAllAtomix( self, cluster, killRemoveMax, stopAtomix ):
@@ -218,24 +218,36 @@
         return cluster.killOnos( killRemoveMax, stopOnos )
 
     def createApplyCell( self, cluster, newCell, cellName, cellApps,
-                         mininetIp, useSSH, ips, installMax=False ):
+                         mininetIp, useSSH, onosIps, installMax=False,
+                         atomixClusterSize=None ):
         """
         Description:
             create new cell ( optional ) and apply it. It will also verify the
             cell.
-        Required:
+        Required Arguments:
             * cluster - the cluster driver that will be used.
             * newCell - True for making a new cell and False for not making it.
             * cellName - The name of the cell.
             * cellApps - The onos apps string.
             * mininetIp - Mininet IP address.
             * useSSH - True for using ssh when creating a cell
-            * ips - ip( s ) of the node( s ).
+            * onosIps - ip( s ) of the ONOS node( s ).
+        Optional Arguments:
+            * installMax
+            * atomixClusterSize - The size of the atomix cluster. Defaults to same
+                as ONOS Cluster size
         Returns:
             Returns main.TRUE if it successfully executed.
         """
+        if atomixClusterSize is None:
+            atomixClusterSize = len( cluster.runningNodes )
+        atomixClusterSize = int( atomixClusterSize )
+        cluster.setAtomixNodes( atomixClusterSize )
+        atomixIps = [ node.ipAddress for node in cluster.atomixNodes ]
+        main.log.info( "Atomix Cluster Size = {} ".format( atomixClusterSize ) )
         if newCell:
-            cluster.createCell( cellName, cellApps, mininetIp, useSSH, ips )
+            cluster.createCell( cellName, cellApps, mininetIp, useSSH, onosIps,
+                                atomixIps, installMax )
         main.step( "Apply cell to environment" )
         stepResult = cluster.applyCell( cellName )
         utilities.assert_equals( expect=main.TRUE,
@@ -252,7 +264,7 @@
         Required:
             * cluster - a cluster driver that will be used.
             * uninstallMax - True for uninstalling max number of nodes
-            False for uninstalling the current running nodes.
+                False for uninstalling the current running nodes.
         Returns:
             Returns main.TRUE if it successfully uninstalled.
         """
@@ -300,13 +312,12 @@
                                  onfail="Failed to create ONOS package" )
         return packageResult
 
-    def installAtomix( self, cluster, installMax, parallel=True ):
+    def installAtomix( self, cluster, parallel=True ):
         """
         Description:
             Installing atomix and verify the result
         Required:
             * cluster - the cluster driver that will be used.
-            * installMax - True for installing max number of nodes
             False for installing current running nodes only.
         Returns:
             Returns main.TRUE if it successfully installed
@@ -314,7 +325,7 @@
         main.step( "Installing Atomix" )
         atomixInstallResult = main.TRUE
 
-        cluster.installAtomix( installMax, parallel )
+        cluster.installAtomix( parallel )
         utilities.assert_equals( expect=main.TRUE,
                                  actual=atomixInstallResult,
                                  onpass="Successfully installed Atomix",
@@ -360,8 +371,8 @@
         secureSshResult = cluster.ssh()
         utilities.assert_equals( expect=main.TRUE,
                                  actual=secureSshResult,
-                                 onpass="Test step PASS",
-                                 onfail="Test step FAIL" )
+                                 onpass="Secured ONOS ssh",
+                                 onfail="Failed to secure ssh" )
         return secureSshResult
 
     def checkOnosService( self, cluster ):
@@ -447,7 +458,7 @@
                                       False,
                                       args = [ apps ],
                                       sleep=5,
-                                      attempts=9 )
+                                      attempts=90 )
 
         utilities.assert_equals( expect=True,
                                  actual=stepResult,
@@ -468,9 +479,10 @@
 
     def ONOSSetUp( self, cluster, hasMultiNodeRounds=False, startOnos=True, newCell=True,
                    cellName="temp", cellApps="drivers", appPrefix="org.onosproject.",
-                   mininetIp="", removeLog=False, extraApply=None, applyArgs=None,
-                   extraClean=None, cleanArgs=None, skipPack=False, installMax=False, useSSH=True,
-                   killRemoveMax=True, stopAtomix=False, stopOnos=False, installParallel=True, cellApply=True,
+                   mininetIp="", extraApply=None, applyArgs=None,
+                   extraClean=None, cleanArgs=None, skipPack=False, installMax=False,
+                   atomixClusterSize=None, useSSH=True, killRemoveMax=True, stopAtomix=False,
+                   stopOnos=False, installParallel=True, cellApply=True,
                    includeCaseDesc=True ):
         """
         Description:
@@ -487,26 +499,28 @@
                 setting up ssh to the onos
                 checking the onos service
                 starting onos
-        Required:
+        Required Arguments:
             * cluster - the cluster driver that will be used.
-            * hasMultiNodeRouds - True if the test is testing different set of nodes
+        Optional Arguments:
+            * hasMultiNodeRounds - True if the test is testing different set of nodes
             * startOnos - True if wish to start onos.
             * newCell - True for making a new cell and False for not making it.
             * cellName - Name of the cell that will be used.
             * cellApps - The cell apps string. Will be overwritten by main.apps if it exists
             * appPrefix - Prefix of app names. Will use "org.onosproject." by default
             * mininetIp - Mininet IP address.
-            * removeLog - True if wish to remove raft logs
             * extraApply - Function( s ) that will be called before building ONOS. Default to None.
             * applyArgs - argument of the functon( s ) of the extraApply. Should be in list.
             * extraClean - Function( s ) that will be called after building ONOS. Defaults to None.
             * cleanArgs - argument of the functon( s ) of the extraClean. Should be in list.
             * skipPack - True if wish to skip some packing.
             * installMax - True if wish to install onos max number of nodes
-            False if wish to install onos of running nodes only
+                False if wish to install onos of running nodes only
+            * atomixClusterSize - The number of Atomix nodes in the cluster.
+                Defaults to None which will be the number of OC# nodes in the cell
             * useSSH - True for using ssh when creating a cell
             * killRemoveMax - True for removing/killing max number of nodes. False for
-            removing/killing running nodes only.
+                removing/killing running nodes only.
             * stopAtomix - True if wish to stop atomix before killing it.
             * stopOnos - True if wish to stop onos before killing it.
         Returns:
@@ -519,7 +533,7 @@
             main.caseExplanation = "Set up ONOS with " + str( cluster.numCtrls ) + \
                                    " node(s) ONOS cluster"
 
-        main.log.info( "NODE COUNT = " + str( cluster.numCtrls ) )
+        main.log.info( "ONOS cluster size = " + str( cluster.numCtrls ) )
         cellResult = main.TRUE
         if cellApply:
             try:
@@ -539,15 +553,13 @@
             cellResult = self.createApplyCell( cluster, newCell,
                                                cellName, apps,
                                                mininetIp, useSSH,
-                                               tempOnosIp, installMax )
+                                               tempOnosIp, installMax,
+                                               atomixClusterSize )
 
         atomixKillResult = self.killingAllAtomix( cluster, killRemoveMax, stopAtomix )
         onosKillResult = self.killingAllOnos( cluster, killRemoveMax, stopOnos )
         killResult = atomixKillResult and onosKillResult
 
-        if removeLog:
-            main.log.info("Removing raft logs")
-            main.ONOSbench.onosRemoveRaftLogs()
         atomixUninstallResult = self.uninstallAtomix( cluster, killRemoveMax )
         onosUninstallResult = self.uninstallOnos( cluster, killRemoveMax )
         uninstallResult = atomixUninstallResult and onosUninstallResult
@@ -557,7 +569,7 @@
         if not skipPack:
             packageResult = self.buildOnos(cluster)
 
-        atomixInstallResult = self.installAtomix( cluster, installMax, installParallel )
+        atomixInstallResult = self.installAtomix( cluster, installParallel )
         onosInstallResult = self.installOnos( cluster, installMax, installParallel )
         installResult = atomixInstallResult and onosInstallResult