Merge pull request #124 from opennetworkinglab/FuncPlatform

Func platform
diff --git a/TestON/tests/FuncPlatform/Dependency/App.py b/TestON/tests/FuncPlatform/Dependency/App.py
index f6fecd6..5410c32 100644
--- a/TestON/tests/FuncPlatform/Dependency/App.py
+++ b/TestON/tests/FuncPlatform/Dependency/App.py
@@ -1,3 +1,8 @@
+"""
+Methods related to application interaction
+
+"""
+
 
 def __init__( self ):
     self.ip = '127.0.0.1'
@@ -10,27 +15,48 @@
         nodeToActivateFrom = range( 0, nodes )
     """
     if isinstance( apps, ( int, basestring ) ):
-        main.log.error( "Please pass in a list of strings for args" )
+        main.log.error( 'Please pass in a list of strings for args' )
         return main.FALSE
 
     if not isinstance( nodeToActivateFrom, ( int ) ) or \
             nodeToActivateFrom < 0:
-        main.log.error( "Incorrect node specified" )
+        main.log.error( 'Incorrect node specified' )
         return main.FALSE
 
+    # TODO: Start log capture and listen for exceptions
+    #       and errors. Also investigate possible keywords
+    #       to listen for when activating applications
+    
+
     for app in apps:
         # Check if app str in appList is in the main scope
         # definition main.appList
         if app not in main.appList:
-            main.log.error( "Invalid app name given" )
+            main.log.error( 'Invalid app name given' )
             return main.FALSE
-      
-        # NOTE: assumes node 1 is always activating application
-        appOutput = main.CLIs[nodeToActivateFrom].activateApp( 
+     
+        try:
+            # NOTE: assumes node 1 is always activating application
+            appOutput = main.CLIs[nodeToActivateFrom].activateApp( 
                 main.appList[app] ) 
+        except KeyError:
+            main.log.error( 'There was an error with the key '+
+                    str(app) + '. Check the appList dictionary' )
+            return main.FALSE
+        except Exception:
+            main.log.error( 'Uncaught exception error while ' +
+                    'activating applications: ' + str(app) )
+            return main.FALSE
 
     return main.TRUE
 
+def deactivate( apps, nodeToDeactivateFrom=0 ):
+    """
+    Deactivate specified applications from node specified
+
+    """
+    main.log.report( 'deactivate implment me' )
+
 def isAppInstallSuccess():
     """
     Check the app list across clusters to determine
@@ -38,5 +64,5 @@
 
     """
 
-    main.log.report( "isAppInstallSuccess" )
+    main.log.report( 'isAppInstallSuccess implement me' )
 
diff --git a/TestON/tests/FuncPlatform/Dependency/Logger.py b/TestON/tests/FuncPlatform/Dependency/Logger.py
new file mode 100644
index 0000000..22d3f61
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Logger.py
@@ -0,0 +1,57 @@
+
+def __init__( self ):
+    self.ip = '127.0.0.1'
+
+def checkOnosLog( nodeIp, option='',
+        outputType=0):
+    """
+    Listens to the log for any Errors and Exceptions.
+
+    Runs 'onos-check-logs <option>'
+    This script only returns if there are any errors
+    or exceptions
+
+    outputType
+        0: Return output of log
+        1: Return (#Errors, #Exceptions, #Warn)
+
+    """
+    if not isinstance( option, basestring ):
+        main.log.error( 'Incorrect grep format specified' )
+        return main.FALSE
+
+    try:
+        main.log.info( 'Starting Onos-log listening for '+
+                str(option) )
+        cmd = 'onos-check-logs ' + str(nodeIp) + ' old'
+        if outputType == 0:
+            main.ONOSbench.handle.sendline( cmd )
+            main.ONOSbench.handle.expect( cmd )
+            main.ONOSbench.handle.expect('\$')
+            logResult = main.ONOSbench.handle.before
+            return logResult
+        elif outputType == 1:
+            # Important in assertion criteria
+            # to determine how much warn / error is 
+            # acceptable
+            return 'Implement option 1'
+        else:
+            main.log.error( 'Incorrect outputType specified' )
+            return main.FALSE
+
+    except Exception:
+        main.log.exception( self.name + ': Uncaught exception' ) 
+        main.cleanup()
+        main.exit()
+
+def setLogLevel( level ):
+    """
+    Set the log level of onos
+    """
+    main.log.info( 'setLogLevel implement me' )
+
+def getLogReport( nodeIp, searchTerms ):
+    """
+    Refer to CLI driver for 'logReport'
+    """
+    main.log.info( 'getLogReport - implement me!' )
diff --git a/TestON/tests/FuncPlatform/Dependency/Shutdown.py b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
new file mode 100644
index 0000000..ba77bbd
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
@@ -0,0 +1,19 @@
+
+def __init__( self ):
+    self.ip = '127.0.0.1'
+
+def killOnosNodes( nodeIps ):
+    """
+    Kill all components of Onos on 
+    given list of ips
+    
+    Ex) nodeIps = ['10.0.0.1', '10.0.0.2']
+    """
+    killResult = main.TRUE
+    
+    for node in nodeIps:
+        killResult = killResult and main.ONOSbench.onosDie( node )
+        if killResult == main.TRUE:
+            main.log.info( str(node) + ' was killed' ) 
+
+    return killResult
diff --git a/TestON/tests/FuncPlatform/Dependency/Startup.py b/TestON/tests/FuncPlatform/Dependency/Startup.py
index 3096eee..f3f90a9 100644
--- a/TestON/tests/FuncPlatform/Dependency/Startup.py
+++ b/TestON/tests/FuncPlatform/Dependency/Startup.py
@@ -4,11 +4,13 @@
 
 Guidelines:
     * Group sequential functionalities together
-    * Methods should not prohibit cross platform execution
+    * Methods should not be dependent on platform
     * Return main.TRUE on success or comprehensive error message 
       on failure (TBD)
+    * All methods should be consistent in expected behavior
 """
 import time
+import json
 
 def __init__( self ):
     self.ip = '127.0.0.1' 
@@ -46,6 +48,10 @@
         * Force install ONOS package
         * Start ONOS service
         * Start ONOS cli
+    
+    Also verifies that Onos is up and running by 
+    'isup' driver function which executs 
+    'onos-wait-for-start'
     """
 
     # NOTE: leave out create cell file until bug addressed
@@ -69,8 +75,145 @@
     for node in range( 0, numNodes ):
         cli = cli and main.CLIs[node].startOnosCli( onosIps[node] )
 
-    if sc and vc and op and oi and iu and cli == main.TRUE:
+    # Check if all nodes are discovered correctly using
+    # 'nodes' command in Onos Cli
+    na = main.TRUE
+    try:
+        nodeCmdJson = json.loads( main.CLIs[0].nodes() )
+        for node in nodeCmdJson:
+            if node['state'] != 'ACTIVE':
+                main.log.warn( str( node['id'] ) + 
+                        ' Node is not in ACTIVE state.' )
+                na = main.FALSE
+        if na != main.FALSE:
+            main.log.info( 'All nodes discovered successfully' )
+    except Exception:
+        main.log.error( 'nodes command did not execute properly' )
+        return main.FALSE
+
+    if sc and vc and op and oi and iu and cli and na == main.TRUE:
         return main.TRUE
     else:
         return main.FALSE
 
+def installOnosFromTar( wgetAddr, nodeIps ):
+    """
+    Install Onos directly from tar.gz file.
+    Due to the nature of the specific steps required 
+    to startup Onos in this fashion, all commands
+    required to start Onos from tar.gz will be
+    grouped in this method. 
+
+    1) wget latest onos tar.gz on onos node
+    2) untar package
+    3) specify onos-config cluster
+    4) start onos via onos-service
+    5) form onos cluster using onos-form-cluster
+    6) check for successful startup
+
+    Specify the download link for the tar.gz.
+    Provide a list of nodeIps
+
+    Ex) wgetAddr = 'https://mytargzdownload.com/file.tar.gz'
+        nodeIps = ['10.0.0.1', '10.0.0.2']
+    """
+    if isinstance( nodeIps, ( int, basestring ) ):
+        main.log.error( 'Please pass in a list of string nodes' )
+        return main.FALSE
+
+    # Obtain filename from provided address
+    # assumes that filename is separated by '/' character
+    f_name = ''
+    addr = str( wgetAddr ).split('/')
+    for phrase in addr:
+        if 'tar.gz' in phrase:
+            f_name = str( phrase )
+            main.log.info( 'Using ' + f_name + ' as file' ) 
+
+    clusterCount = len( nodeIps )
+
+    main.log.info( 'Initiating Onos installation sequence ' +
+            'using tar.gz ... This may take a few minutes' )
+
+    for node in range( 0, clusterCount ):
+        # Use the wgetAddr to download a new tar.gz from server
+        try:
+            main.ONOSnode[node].handle.sendline( 'wget ' + wgetAddr )
+            main.ONOSnode[node].handle.expect( 'saved' )
+            main.ONOSnode[node].handle.expect( '\$' )
+            main.log.info( 'Successfully downloaded tar.gz ' +
+                    'on node: ' + str( main.ONOSips[node] ) ) 
+        except Exception:
+            # NOTE: Additional exception may be appropriate 
+            main.log.error( 'Uncaught exception while ' +
+                    'downloading Onos tar.gz: ' + 
+                    main.ONOSnode[node].handle.before )
+            return main.FALSE
+
+    for node in range( 0, clusterCount ):
+        # Untar files on all nodes, then enter the 
+        # newly created directory
+        try:
+            main.ONOSnode[node].handle.sendline( 'tar zxvf ' + f_name )
+            # Verbose output of tar will contain some onos returns
+            main.ONOSnode[node].handle.expect( 'onos' )
+            main.ONOSnode[node].handle.expect( '\$' )
+            # NOTE: Making an assumption here: 
+            #       the directory created by tar file has a name that
+            #       starts with 'onos-' followed by version number
+            #       '1.2.0'. If this is NOT true, this method must
+            #       be changed to enter the correct directory.
+            #       The directory name is currently dynamic 
+            #       and depends on the day at which you downloaded
+            #       the tar file
+            # Enter onos- wildcard to disregard its version / date 
+            #       suffix
+            main.ONOSnode[node].handle.sendline( 'cd onos-*' )
+            main.ONOSnode[node].handle.expect( '\$' )
+            
+            main.ONOSnode[node].handle.sendline( 'pwd' )
+            main.ONOSnode[node].handle.expect( 'pwd' )
+            main.ONOSnode[node].handle.expect( '\$' )
+            pwd = main.ONOSnode[node].handle.before 
+            if 'onos' in str(pwd):
+                main.log.info( 'tar zxvf ' + f_name + ' successful ' +
+                        'on node ' + str( main.ONOSips[node] ) )
+
+        except Exception:
+            main.log.error( 'Uncaught exception while executing ' +
+                    'tar zxvf of ' + f_name + ': ' +
+                    main.ONOSnode[node].handle.before +
+                    main.ONOSnode[node].handle.after)
+            return main.FALSE
+
+    for node in range( 0, clusterCount ):
+        try:
+            main.ONOSnode[node].handle.sendline( 'bin/onos-service '+
+                    'server &' )
+            # Send some extra characters to run the process
+            main.ONOSnode[node].handle.sendline( '' )
+            main.ONOSnode[node].handle.sendline( '' )
+            main.ONOSnode[node].handle.expect( '\$' )
+        except Exception:
+            main.log.error( 'Uncaught exception while executing ' +
+                    'onos-service server command ' + 
+                    str( main.ONOSnode[node].handle.before ) )
+            return main.FALSE
+    
+    iu = main.TRUE
+    for node in nodeIps:
+        iu = iu and main.ONOSbench.isup( node )
+
+    if iu == main.TRUE: 
+        return main.TRUE
+    else:
+        return main.FALSE
+    
+def addAndStartOnosNode( nodeIps ):
+    """
+    A scale-out scenario that adds specified list of 
+    nodes and starts those instances.
+
+    Ex) nodeIps = ['10.0.0.2', '10.0.0.3', 10.0.0.4']
+    """
+    main.log.info( 'addAndStartOnosNode implement me!' )
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.params b/TestON/tests/FuncPlatform/FuncPlatform.params
index eb274ae..3895660 100644
--- a/TestON/tests/FuncPlatform/FuncPlatform.params
+++ b/TestON/tests/FuncPlatform/FuncPlatform.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,2,3</testcases>
+    <testcases>1,2,3,4</testcases>
     <DEP>
         <startupSrc>
             /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Startup.py
@@ -14,6 +14,24 @@
         <appClassName>
             App
         </appClassName>
+    
+        <logSrc>
+            /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Logger.py
+        </logSrc>
+        <logClassName>
+            Log
+        </logClassName>
+   
+        <shutdownSrc>
+            /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Shutdown.py
+        </shutdownSrc>
+        <shutdownClassName>
+            Shutdown
+        </shutdownClassName>
+
+        <targz>
+        http://downloads.onosproject.org/nightly/onos-1.2.0.latest-NIGHTLY.tar.gz
+        </targz>
     </DEP> 
 
     <CTRL>
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.py b/TestON/tests/FuncPlatform/FuncPlatform.py
index 6bcb06c..60b1aa0 100644
--- a/TestON/tests/FuncPlatform/FuncPlatform.py
+++ b/TestON/tests/FuncPlatform/FuncPlatform.py
@@ -19,6 +19,10 @@
         3. Activate application Y
         4. Deactivate application X
 
+The ideal platform test script should have incredible
+robustness to possible exceptions and report the most
+useful error messages. 
+
 contributers to contact for help:
 andrew@onlab.us
 """
@@ -30,7 +34,13 @@
     def CASE1( self, main ):
         """
         Main scope initialization case
+        Must include to run any other test cases
         """   
+        import imp 
+
+        # NOTE: Hardcoded application name subject to change
+        #       closely monitor and make changes when necessary
+        #       (or implement ways to dynamically get names)
         main.appList = { 
             'bgprouter' : 'org.onosproject.bgprouter',
             'config' : 'org.onosproject.config',
@@ -46,23 +56,59 @@
             'mobility' : 'org.onosproject.mobility',
             'netconf' : 'org.onosproject.netconf', 
             'null' : 'org.onosproject.null',
-            'optical' : 'org.onosproject.optical'
+            'optical' : 'org.onosproject.optical',
+            'pcep' : 'org.onosproject.pcep',
+            'proxyarp' : 'org.onosproject.proxyarp',
+            'reactive.routing' : 'org.onosproject.reactive.routing',
+            'sdnip' : 'org.onosproject.sdnip',
+            'segmentrouting' : 'org.onosproject.segmentrouting',
+            'tunnel' : 'org.onosproject.tunnel',
+            'virtualbng' : 'org.onosproject.virtualbng',
+            'xosintegration' : 'org.onosproject.xosintegration'
             } 
         # List of ONOS ip's specififed in params
         main.ONOSips = [] 
         main.CLIs = []
+        main.ONOSnode = []
 
         for node in range( 0, int(main.params['CTRL']['num']) ):
             main.ONOSips.append( main.params['CTRL']['ip'+str(node+1)] )
             main.CLIs.append(
                     getattr( main, 'ONOS' + str(node+1) + 'cli' ) )
+            main.ONOSnode.append(
+                    getattr( main, 'ONOS' + str(node+1) ) )
        
-    def CASE2( self, main ):
-        import time
-        import imp
-
+        # Application source and name definitions
         startupSrc = main.params['DEP']['startupSrc']
         startupClassName = main.params['DEP']['startupClassName']
+        
+        appClassName = main.params['DEP']['appClassName']
+        appSrc = main.params['DEP']['appSrc']
+
+        logClassName = main.params['DEP']['logClassName']
+        logSrc = main.params['DEP']['logSrc']
+
+        shutdownClassName = main.params['DEP']['shutdownClassName']
+        shutdownSrc = main.params['DEP']['shutdownSrc']
+
+        # Importing dependency class(es)
+        # Refer to source files in Dependency folder to
+        # make changes to its respective methods
+        # Be weary of naming collisions
+        try:
+            main.startup = imp.load_source( startupClassName, startupSrc )
+            main.app = imp.load_source( appClassName, appSrc )
+            main.onosLog = imp.load_source( logClassName, logSrc )
+            main.shutdown = imp.load_source( shutdownClassName, shutdownSrc )
+        except ImportError:
+            main.log.error( 'Error importing class file(s). Please ' +
+                    'check file location' )
+            main.cleanup()
+            main.exit()
+
+    def CASE2( self, main ):
+        import time
+
         cellName = main.params['CELL']['name']
         appStr = main.params['CELL']['appStr']
         benchIp = main.params['BENCH']['ip']
@@ -70,21 +116,10 @@
         gitPull = main.params['GIT']['pull']
         mnIp = main.params['MN']['ip']
 
-        # importing dependency class(es)
-        # Refer to source files in Dependency folder to
-        # make changes to its respective methods
-        try:
-            startup = imp.load_source( startupClassName, startupSrc )
-        except ImportError:
-            main.log.error( "Error importing class " +
-                    str(startupClassName) + " from " + str(startupSrc) )
-            main.cleanup()
-            main.exit()
-
         main.case( 'Setup environment and install ONOS' )
         if gitPull == 'on': 
             main.step( 'Git pull and clean install' )
-            gitPullResult = startup.gitPullAndMci( branchName )
+            gitPullResult = main.startup.gitPullAndMci( branchName )
             utilities.assert_equals( expect=main.TRUE,
                         actual=gitPullResult,
                         onpass='Git pull and install successful',
@@ -92,7 +127,7 @@
                             str(gitPullResult) )
         
         main.step( 'Initiate ONOS startup sequence' )    
-        startupResult = startup.initOnosStartupSequence(
+        startupResult = main.startup.initOnosStartupSequence(
                 cellName, appStr, benchIp, mnIp, main.ONOSips )
         utilities.assert_equals( expect=main.TRUE,
                         actual=startupResult,
@@ -102,31 +137,58 @@
         
     def CASE3( self, main ):
         import time
-        import imp
 
         main.case( 'Activate applications and check installation' )
-        # Activate applications and check consistency 
-        # across clusters
-        appClassName = main.params['DEP']['appClassName']
-        appSrc = main.params['DEP']['appSrc']
+       
+        # NOTE: Test only
+        # Unceremoniously kill onos 2 
+        main.ONOSbench.onosDie( '10.128.174.2' )
+        
+        time.sleep( 30 )
 
-        # Import application file to use its methods
-        try:
-            app = imp.load_source( appClassName, appSrc )
-        except ImportError:
-            main.log.error( "Error importing class " +
-                    str(startupClassName) + " from " + str(startupSrc) )
-            main.cleanup()
-            main.exit()
+        main.step( 'Sample Onos log check' )
+        logResult = main.onosLog.checkOnosLog( main.ONOSips[0] )
+        main.log.info( logResult )
+        # TODO: Define pass criteria
+        utilities.assert_equals( expect=main.TRUE,
+                actual=main.TRUE,
+                onpass= 'Logging successful',
+                onfail= 'Logging failed ' )
 
         # Sample app activation
         main.step( 'Activating applications metrics and fwd' ) 
         appList = ['metrics', 'fwd']
-        appResult = app.activate( appList ) 
+        appResult = main.app.activate( appList ) 
         utilities.assert_equals( expect=main.TRUE,
                 actual=appResult,
                 onpass= 'App activation of ' + str(appList) + ' successful',
                 onfail= 'App activation failed ' + str(appResult) )
 
+    def CASE4( self, main ):
+        """
+        Download ONOS tar.gz built from latest nightly
+        (following tutorial on wiki) and run ONOS directly on the
+        instance
+        """
+        import imp
+
+        targz = main.params['DEP']['targz']
+        clusterCount = main.params['CTRL']['num']
+       
+        main.case( 'Install ONOS from onos.tar.gz file' )
+
+        main.step( 'Killing all ONOS instances previous started' )
+        killResult = main.shutdown.killOnosNodes( main.ONOSips )
+        utilities.assert_equals( expect=main.TRUE,
+                actual = killResult,
+                onpass = 'All Onos nodes successfully killed',
+                onfail = 'Onos nodes were not successfully killed' )
+
+        main.step( 'Starting ONOS using tar.gz on all nodes' )
+        installResult = main.startup.installOnosFromTar( targz, main.ONOSips )
+        utilities.assert_equals( expect=main.TRUE,
+                actual = installResult,
+                onpass= 'Onos tar.gz installation successful',
+                onfail= 'Onos tar.gz installation failed' )
 
 
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.topo b/TestON/tests/FuncPlatform/FuncPlatform.topo
index fcc087d..f19e8f8 100644
--- a/TestON/tests/FuncPlatform/FuncPlatform.topo
+++ b/TestON/tests/FuncPlatform/FuncPlatform.topo
@@ -14,7 +14,7 @@
             <host>10.128.174.1</host>
             <user>admin</user>
             <password></password>
-            <type>OnosCliDriver</type>
+            <type>OnosDriver</type>
             <connect_order>3</connect_order>
             <COMPONENTS> </COMPONENTS>
         </ONOS1>
@@ -23,7 +23,7 @@
             <host>10.128.174.2</host>
             <user>admin</user>
             <password></password>
-            <type>OnosCliDriver</type>
+            <type>OnosDriver</type>
             <connect_order>4</connect_order>
             <COMPONENTS> </COMPONENTS>
         </ONOS2>
@@ -32,7 +32,7 @@
             <host>10.128.174.3</host>
             <user>admin</user>
             <password></password>
-            <type>OnosCliDriver</type>
+            <type>OnosDriver</type>
             <connect_order>5</connect_order>
             <COMPONENTS> </COMPONENTS>
         </ONOS3>