Update tests for aether pods
- Update test for QA-POD
- SRStaging for testing connecting to Staging pod
- Add some functions for a kubernetes deployed cluster
- Connect to ONOS nodes with kubernetes
- Add option to connect to components through jump hosts
- Fixes for installing ONOS in custom locations
- Invoke python2 instead of python
- If using an ssh agent, also use that for pexpect ssh sessions,
E.G. Jenkins initiated tests
Change-Id: I1fc345c8eab60a5b00c17e6ed677a63489a74a19
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
new file mode 100644
index 0000000..cb37498
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.params
@@ -0,0 +1,80 @@
+<PARAMS>
+ <testcases>7</testcases>
+
+ <GRAPH>
+ <nodeCluster>staging</nodeCluster>
+ <builds>20</builds>
+ <jobName>SRStaging</jobName>
+ <branch>master</branch>
+ </GRAPH>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <useCommonConf>False</useCommonConf>
+ <useCommonTopo>True</useCommonTopo>
+ <useBmv2>True</useBmv2>
+ <bmv2SwitchType>stratum</bmv2SwitchType>
+ <switchPrefix></switchPrefix>
+ <stratumRoot>~/stratum</stratumRoot>
+ <topology>trellis_fabric.py</topology>
+ <lib>routinglib.py,trellislib.py,stratum.py</lib>
+ </DEPENDENCY>
+
+ <jsonFileSuffix>.hw</jsonFileSuffix>
+
+ <persistent_setup>True</persistent_setup>
+
+ <kubernetes>
+ <appName>onos-tost-onos-classic</appName>
+ <namespace>tost</namespace>
+ </kubernetes>
+
+ <PERF>
+ <traffic_host>Host1 Host2 Host3</traffic_host>
+ <traffic_cmd_arguments> -u -b 20M -t 20</traffic_cmd_arguments>
+
+ <pcap_host>ng40vm</pcap_host>
+ <pcap_cmd_arguments>-t e -F pcap -s 100 </pcap_cmd_arguments>
+
+ </PERF>
+ <ONOS_Logging>
+ <org.onosproject.p4runtime.ctl.client>DEBUG</org.onosproject.p4runtime.ctl.client>
+ <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+ <org.onosproject.gnmi.ctl>TRACE</org.onosproject.gnmi.ctl>
+ </ONOS_Logging>
+
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,fpm,lldpprovider,hostprovider,netcfghostprovider,drivers.bmv2,org.opencord.fabric-tofino,pipelines.fabric,org.stratumproject.fabric-tna,drivers.barefoot,segmentrouting,t3</cellApps>
+ </ENV>
+
+ <EXTERNAL_APPS>
+ </EXTERNAL_APPS>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ </timers>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+ <TOPO>
+ <switchNum>4</switchNum>
+ <linkNum>16</linkNum>
+ </TOPO>
+
+ <ALARM>
+ <minPassPercent>100</minPassPercent>
+ </ALARM>
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
new file mode 100644
index 0000000..124fd50
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.py
@@ -0,0 +1,91 @@
+class SRStaging:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ main.case("Testing connections")
+ main.persistentSetup = True
+ def CASE7( self, main ):
+ """
+ Tests connectivity between two untagged hosts
+ (Ports are configured as vlan-untagged)
+
+ Sets up 3 ONOS instance
+ Start 2x2 leaf-spine topology
+ Pingall
+ """
+ try:
+ from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+ except ImportError:
+ main.log.error( "SRStagingTest not found. Exiting the test" )
+ main.cleanAndExit()
+ try:
+ main.funcs
+ except ( NameError, AttributeError ):
+ main.funcs = SRStagingTest()
+ # Load kubeconfig
+ # Setup ssh tunnel
+ # connect to ONOS CLI
+
+
+ main.funcs.setupTest( main,
+ test_idx=7,
+ topology='2x2staging',
+ onosNodes=3,
+ description="Developing tests on the staging pod" )
+ srcComponentNames = main.params[ 'PERF' ][ 'traffic_host' ].split()
+ srcComponentList = []
+ for name in srcComponentNames:
+ srcComponentList.append( getattr( main, name ) )
+ dstComponent = getattr( main, main.params[ 'PERF' ][ 'pcap_host' ] )
+
+ main.downtimeResults = {}
+
+
+ # TODO: MOVE TO CONFIG FILE
+ device = "device:leaf2"
+ port1 = "268"
+ port2 = "284"
+ port3 = "260"
+ port4 = "276"
+
+ descPrefix = "Upstream_Leaf_Spine_Portstate"
+ # TODO: Move most of this logic into linkDown/linkUp
+ ## First Link Down
+ shortDesc = descPrefix + "-Failure1"
+ longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port1 )
+ main.funcs.linkDown( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Second Link Down
+ shortDesc = descPrefix + "-Failure2"
+ longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port2 )
+ main.funcs.linkDown( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## First Link Up
+ # TODO Check these are set correctly
+ shortDesc = descPrefix + "-Recovery1"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port1 )
+ main.funcs.linkUp( device, port1, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Second Link Up
+ shortDesc = descPrefix + "-Recovery2"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port2 )
+ main.funcs.linkUp( device, port2, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Third Link Down
+ shortDesc = descPrefix + "-Failure3"
+ longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port3 )
+ main.funcs.linkDown( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Forth Link Down
+ shortDesc = descPrefix + "-Failure4"
+ longDesc = "%s Failure: Bring down %s/%s" % ( descPrefix, device, port4 )
+ main.funcs.linkDown( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Third Link Up
+ shortDesc = descPrefix + "-Recovery3"
+ longDesc = "%s Recovery: Bring upn %s/%s" % ( descPrefix, device, port3 )
+ main.funcs.linkUp( device, port3, srcComponentList, dstComponent, shortDesc, longDesc )
+ ## Forth Link Up
+ shortDesc = descPrefix + "-Recovery4"
+ longDesc = "%s Recovery: Bring up %s/%s" % ( descPrefix, device, port4 )
+ main.funcs.linkUp( device, port4, srcComponentList, dstComponent, shortDesc, longDesc )
+
+ main.log.warn( main.downtimeResults )
+ import json
+ main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+ main.funcs.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
new file mode 100644
index 0000000..8c25811
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRStaging.topo
@@ -0,0 +1,218 @@
+<TOPOLOGY>
+ <COMPONENT>
+ <ONOScell>
+ <host>localhost</host> # ONOS "bench" machine
+ <user>jenkins</user>
+ <password></password>
+ <type>OnosClusterDriver</type>
+ <connect_order>50</connect_order>
+ <jump_host></jump_host>
+ <home>~/Projects/onos/</home> # defines where onos home is on the build machine. Defaults to "~/onos/" if empty.
+ <COMPONENTS>
+ <kubeConfig>~/.kube/stg-ace-menlo</kubeConfig> # If set, will attempt to use this file for setting up port-forwarding
+ <useDocker>True</useDocker> # Whether to use docker for ONOS nodes
+ <docker_prompt>\$</docker_prompt>
+ <cluster_name></cluster_name> # Used as a prefix for cluster components. Defaults to 'ONOS'
+ <diff_clihost>True</diff_clihost> # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
+ <karaf_username>karaf</karaf_username>
+ <karaf_password>karaf</karaf_password>
+ <web_user>sdn</web_user>
+ <web_pass>rocks</web_pass>
+ <rest_port></rest_port>
+ <prompt></prompt> # TODO: we technically need a few of these, one per component
+ <onos_home>~/Projects/onos/</onos_home> # defines where onos home is on the target cell machine. Defaults to entry in "home" if empty.
+ <nodes> 3 </nodes> # number of nodes in the cluster
+ </COMPONENTS>
+ </ONOScell>
+
+ <SwitchLeaf1>
+ <host>10.32.4.132</host>
+ <user>root</user>
+ <password>onl</password>
+ <type>StratumOSSwitchDriver</type>
+ <connect_order>12</connect_order>
+ <jump_host>NetworkBench</jump_host>
+ <COMPONENTS>
+ <shortName>leaf1</shortName>
+ <port1></port1>
+ <link1></link1>
+ <port2></port2>
+ <link2></link2>
+ <onosConfigPath></onosConfigPath>
+ <onosConfigFile></onosConfigFile>
+ </COMPONENTS>
+ </SwitchLeaf1>
+
+ <SwitchLeaf2>
+ <host>10.32.4.136</host>
+ <user>root</user>
+ <password>onl</password>
+ <type>StratumOSSwitchDriver</type>
+ <connect_order>13</connect_order>
+ <jump_host>NetworkBench</jump_host>
+ <COMPONENTS>
+ <shortName>leaf2</shortName>
+ <port1></port1>
+ <link1></link1>
+ <port2></port2>
+ <link2></link2>
+ <onosConfigPath></onosConfigPath>
+ <onosConfigFile></onosConfigFile>
+ </COMPONENTS>
+ </SwitchLeaf2>
+
+ <SwitchSpine1>
+ <host>10.32.4.130</host>
+ <user>root</user>
+ <password>onl</password>
+ <type>StratumOSSwitchDriver</type>
+ <connect_order>14</connect_order>
+ <jump_host>NetworkBench</jump_host>
+ <COMPONENTS>
+ <shortName>spine1</shortName>
+ <port1></port1>
+ <link1></link1>
+ <port2></port2>
+ <link2></link2>
+ <onosConfigPath></onosConfigPath>
+ <onosConfigFile></onosConfigFile>
+ </COMPONENTS>
+ </SwitchSpine1>
+
+ <SwitchSpine2>
+ <host>10.32.4.134</host>
+ <user>root</user>
+ <password>onl</password>
+ <type>StratumOSSwitchDriver</type>
+ <connect_order>15</connect_order>
+ <jump_host>NetworkBench</jump_host>
+ <COMPONENTS>
+ <shortName>spine2</shortName>
+ <port1></port1>
+ <link1></link1>
+ <port2></port2>
+ <link2></link2>
+ <onosConfigPath></onosConfigPath>
+ <onosConfigFile></onosConfigFile>
+ </COMPONENTS>
+ </SwitchSpine2>
+
+ <Host1>
+ <host>10.32.4.138</host>
+ <user>jenkins</user>
+ <password></password>
+ <type>HostDriver</type>
+ <connect_order>6</connect_order>
+ <jump_host>NetworkBench</jump_host>
+ <COMPONENTS>
+ <mac></mac>
+ <inband>false</inband>
+ <ip>10.32.5.161</ip>
+ <shortName>h1</shortName>
+ <port1></port1>
+ <link1></link1>
+ <interfaceName>enp175s0f0</interfaceName>
+ <routes>
+ <route1>
+ <network></network>
+ <netmask></netmask>
+ <gw></gw>
+ <interface></interface>
+ </route1>
+ </routes>
+ <sudo_required>false</sudo_required>
+ </COMPONENTS>
+ </Host1>
+
+ <Host2>
+ <host>10.32.4.139</host>
+ <user>jenkins</user>
+ <password></password>
+ <type>HostDriver</type>
+ <connect_order>7</connect_order>
+ <jump_host>NetworkBench</jump_host>
+ <COMPONENTS>
+ <mac></mac>
+ <inband>false</inband>
+ <ip>10.32.5.162</ip>
+ <shortName>h2</shortName>
+ <port1></port1>
+ <link1></link1>
+ <interfaceName>enp175s0f0</interfaceName>
+ <routes>
+ <route1>
+ <network></network>
+ <netmask></netmask>
+ <gw></gw>
+ <interface></interface>
+ </route1>
+ </routes>
+ <sudo_required>false</sudo_required>
+ </COMPONENTS>
+ </Host2>
+
+ <Host3>
+ <host>10.32.4.140</host>
+ <user>jenkins</user>
+ <password></password>
+ <type>HostDriver</type>
+ <connect_order>8</connect_order>
+ <jump_host>NetworkBench</jump_host>
+ <COMPONENTS>
+ <mac></mac>
+ <inband>false</inband>
+ <ip>10.32.5.163</ip>
+ <shortName>h3</shortName>
+ <port1></port1>
+ <link1></link1>
+ <interfaceName>enp175s0f0</interfaceName>
+ <routes>
+ <route1>
+ <network></network>
+ <netmask></netmask>
+ <gw></gw>
+ <interface></interface>
+ </route1>
+ </routes>
+ <sudo_required>false</sudo_required>
+ </COMPONENTS>
+ </Host3>
+
+ <ng40vm>
+ <host>10.32.5.6</host>
+ <user>ng40</user>
+ <password>ng40</password>
+ <type>HostDriver</type>
+ <connect_order>8</connect_order>
+ <jump_host>NetworkBench</jump_host>
+ <COMPONENTS>
+ <mac></mac>
+ <inband>false</inband>
+ <ip>10.32.5.6</ip>
+ <shortName>ng40</shortName>
+ <port1></port1>
+ <link1></link1>
+ <interfaceName>ens8</interfaceName>
+ <routes>
+ <route1>
+ <network></network>
+ <netmask></netmask>
+ <gw></gw>
+ <interface></interface>
+ </route1>
+ </routes>
+ </COMPONENTS>
+ </ng40vm>
+
+ <NetworkBench>
+ <host>66.201.42.222</host>
+ <user>jenkins</user>
+ <password></password>
+ <type>NetworkDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </NetworkBench>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
new file mode 100644
index 0000000..5dd43da
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/SRStagingTest.py
@@ -0,0 +1,393 @@
+"""
+Copyright 2017 Open Networking Foundation ( ONF )
+
+Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
+the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
+or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
+
+ TestON is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 2 of the License, or
+ ( at your option ) any later version.
+
+ TestON is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with TestON. If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as run
+import time
+import re
+import json
+import pexpect
+
+class SRStagingTest ():
+
+ def __init__( self ):
+ self.default = ''
+ self.topo = dict()
+ # TODO: Check minFlowCount of leaf for BMv2 switch
+ # (number of spine switch, number of leaf switch, dual-homed, description, minFlowCount - leaf (OvS), minFlowCount - leaf (BMv2))
+ self.topo[ '0x1' ] = ( 0, 1, False, 'single ToR', 28, 20 )
+ self.topo[ '0x2' ] = ( 0, 2, True, 'dual-homed ToR', 37, 37 )
+ self.topo[ '2x2' ] = ( 2, 2, False, '2x2 leaf-spine topology', 37, 32 )
+ self.topo[ '2x2staging' ] = ( 2, 2, True, '2x2 leaf-spine topology', 37, 32 )
+ # TODO: Implement 2x3 topology
+ # topo[ '2x3' ] = ( 2, 3, True, '2x3 leaf-spine topology with dual ToR and single ToR', 28 )
+ self.topo[ '2x4' ] = ( 2, 4, True, '2x4 dual-homed leaf-spine topology', 53, 53 )
+ self.topo[ '2x4' ] = ( 2, 4, True, '2x4 dual-homed leaf-spine topology', 53, 53 )
+ self.switchNames = {}
+ self.switchNames[ '0x1' ] = [ "leaf1" ]
+ self.switchNames[ '2x2' ] = [ "leaf1", "leaf2", "spine101", "spine102" ]
+ main.switchType = "ovs"
+
+ def setupTest( self, main, test_idx, topology, onosNodes, description, vlan = [] ):
+ try:
+ skipPackage = False
+ init = False
+ if not hasattr( main, 'apps' ):
+ init = True
+ run.initTest( main )
+ # Skip onos packaging if the cluster size stays the same
+ if not init and onosNodes == main.Cluster.numCtrls:
+ skipPackage = True
+
+ main.case( '%s, with %s, %s switches and %d ONOS instance%s' %
+ ( description, self.topo[ topology ][ 3 ],
+ main.switchType,
+ onosNodes,
+ 's' if onosNodes > 1 else '' ) )
+
+ main.cfgName = 'CASE%01d%01d' % ( test_idx / 10, ( ( test_idx - 1 ) % 10 ) % 4 + 1 )
+ main.Cluster.setRunningNode( onosNodes )
+ # Set ONOS Log levels
+ # TODO: Check levels before and reset them after
+ run.installOnos( main, skipPackage=skipPackage, cliSleep=5 )
+
+ if hasattr( main, 'Mininet1' ):
+ run.mnDockerSetup( main ) # optionally create and setup docker image
+
+ # Run the test with Mininet
+ mininet_args = ' --spine=%d --leaf=%d' % ( self.topo[ topology ][ 0 ], self.topo[ topology ][ 1 ] )
+ if self.topo[ topology ][ 2 ]:
+ mininet_args += ' --dual-homed'
+ if len( vlan ) > 0 :
+ mininet_args += ' --vlan=%s' % ( ','.join( ['%d' % vlanId for vlanId in vlan ] ) )
+ if main.useBmv2:
+ mininet_args += ' --switch %s' % main.switchType
+ main.log.info( "Using %s switch" % main.switchType )
+
+ run.startMininet( main, 'trellis_fabric.py', args=mininet_args )
+
+ else:
+ # Run the test with physical devices
+ run.connectToPhysicalNetwork( main, hostDiscovery=False ) # We don't want to do host discovery in the pod
+ except Exception as e:
+ main.log.exception( "Error in setupTest" )
+ main.skipCase( result="FAIL", msg=e )
+
+ def startCapturing( self, main, srcList, dst, shortDesc=None, longDesc=None ):
+ """
+ Starts logging, traffic generation, traffic filters, etc before a failure is induced
+ src: the src component that sends the traffic
+ dst: the dst component that receives the traffic
+ """
+ try:
+ # ping right before to make sure arp is cached and sudo is authenticated
+ for src in srcList:
+ src.handle.sendline( "sudo /bin/ping -c 1 %s" % dst.ip_address )
+ try:
+ i = src.handle.expect( [ "password", src.prompt ] )
+ if i == 0:
+ src.handle.sendline( src.pwd )
+ src.handle.expect( src.prompt )
+ except Exception:
+ main.log.error( "Unexpected response from ping" )
+ src.handle.send( '\x03' ) # ctrl-c
+ src.handle.expect( src.prompt )
+ main.log.warn( "%s: %s" % ( src.name, src.handle.before ) )
+ # TODO: Create new components for iperf and tshark?
+ # Also generate more streams with differnt udp ports or some other
+ # method of guranteeing we kill a link with traffic
+ # Start traffic
+ # TODO: ASSERTS
+ main.pingStart = time.time()
+ dstIp = dst.interfaces[0]['ips'][0]
+ for src in srcList:
+ srcIp = src.interfaces[0]['ips'][0]
+ iperfArgs = "%s --bind %s -c %s" % ( main.params[ 'PERF' ][ 'traffic_cmd_arguments' ],
+ srcIp,
+ dstIp )
+ main.log.info( "Starting iperf" )
+ src.handle.sendline( "/usr/bin/iperf %s &> /dev/null &" % iperfArgs )
+ src.handle.expect( src.prompt )
+ # Check path of traffic, to use in failures
+ # TODO: Do we need to add udp port to filter?
+ # TODO: Dynamically find the interface to filter on
+ # Start packet capture
+ pcapFileReceiver = "%s/tshark/%s-%s-tsharkReceiver" % ( "~/TestON",
+ shortDesc if shortDesc else "tshark",
+ dst.name )
+ tsharkArgsReceiver = "%s -i %s -f 'udp && host %s' -w %s" % ( main.params[ 'PERF' ][ 'pcap_cmd_arguments' ],
+ dst.interfaces[0]['name'],
+ dstIp,
+ pcapFileReceiver )
+ commands = [ 'mkdir -p ~/TestON/tshark',
+ 'rm %s' % pcapFileReceiver,
+ 'touch %s' % pcapFileReceiver,
+ 'chmod o=rw %s' % pcapFileReceiver ]
+ for command in commands:
+ dst.handle.sendline( command )
+ dst.handle.expect( dst.prompt )
+ main.log.debug( "%s: %s" % (dst.name, dst.handle.before ) )
+ main.log.info( "Starting tshark on %s " % dst.name )
+ dst.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null &" % tsharkArgsReceiver )
+ dst.handle.expect( dst.prompt )
+
+ for src in srcList:
+ srcIp = src.interfaces[0]['ips'][0]
+ pcapFileSender = "%s/tshark/%s-%s-tsharkSender" % ( "~/TestON",
+ shortDesc if shortDesc else "tshark",
+ src.name )
+ tsharkArgsSender = "%s -i %s -f 'udp && host %s' -w %s" % ( main.params[ 'PERF' ][ 'pcap_cmd_arguments' ],
+ src.interfaces[0]['name'],
+ srcIp,
+ pcapFileSender )
+ # Prepare file with correct permissions
+ commands = [ 'mkdir -p ~/TestON/tshark',
+ 'rm %s' % pcapFileSender,
+ 'touch %s' % pcapFileSender,
+ 'chmod o=rw %s' % pcapFileSender ]
+ for command in commands:
+ src.handle.sendline( command )
+ src.handle.expect( src.prompt )
+ main.log.debug( "%s: %s" % (src.name, src.handle.before ) )
+
+ main.log.info( "Starting tshark on %s " % src.name )
+ for src in srcList:
+ src.handle.sendline( "sudo /usr/bin/tshark %s &> /dev/null &" % tsharkArgsSender )
+ src.handle.expect( src.prompt )
+ # Timestamp used for EVENT START
+ main.eventStart = time.time()
+ # LOG Event start in ONOS logs
+ for ctrl in main.Cluster.active():
+ ctrl.CLI.log( "'%s START'" % longDesc, level="INFO" )
+ except Exception as e:
+ main.log.exception( "Error in startCapturing" )
+ main.skipCase( result="FAIL", msg=e )
+
+ def stopCapturing( self, main, srcList, dst, shortDesc=None, longDesc=None ):
+ try:
+ pcapFileReceiver = "%s/tshark/%s-%s-tsharkReceiver" % ( "~/TestON",
+ shortDesc if shortDesc else "tshark",
+ dst.name )
+ # Timestamp used for EVENT STOP
+ main.eventStop = time.time()
+ # LOG Event stop in ONOS logs
+ for ctrl in main.Cluster.active():
+ ctrl.CLI.log( "'%s STOP'" % longDesc, level="INFO" )
+ # Stop packet capture
+ dst.handle.sendline( 'fg' ) # Bring process to front
+ dst.handle.send( '\x03' ) # send ctrl-c
+ dst.handle.expect( dst.prompt )
+ for src in srcList:
+ src.handle.sendline( 'fg' ) # Bring process to front
+ src.handle.send( '\x03' ) # send ctrl-c
+ src.handle.expect( src.prompt )
+ # Stop traffic
+ for src in srcList:
+ src.handle.sendline( 'fg' ) # Bring process to front
+ src.handle.send( '\x03' ) # send ctrl-c
+ src.handle.expect( src.prompt )
+ main.pingStop = time.time()
+ main.log.warn( "It took %s seconds since we started ping for us to stop pcap" % ( main.pingStop - main.pingStart ) )
+
+ main.downtimeResults[ shortDesc ] = {}
+ for src in srcList:
+ pcapFileSender = "%s/tshark/%s-%s-tsharkSender" % ( "~/TestON",
+ shortDesc if shortDesc else "tshark",
+ src.name )
+ main.downtimeResults[ shortDesc ].update( { src.name: self.analyzePcap( src, pcapFileSender, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False) } )
+ main.downtimeResults[ shortDesc ].update( { "%s-%s" % ( src.name, dst.name ): self.analyzePcap( dst, pcapFileReceiver, "'udp && ip.src == %s'" % src.interfaces[0]['ips'][0], debug=False) } )
+ # Grab pcap
+ senderSCP = main.ONOSbench.scp( src, pcapFileSender, main.logdir, direction="from" )
+ # Grab logs
+ # Grab pcap
+ receiverSCP = main.ONOSbench.scp( dst, pcapFileReceiver, main.logdir, direction="from" )
+ # Grab Write logs on switches
+ # TODO: kubectl cp write-reqs.txt
+
+ except Exception as e:
+ main.log.exception( "Error in stopCapturing" )
+
+ def linkDown( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc ):
+ """"
+ High level function that handles an event including monitoring
+ Arguments:
+ device - String of the device uri in ONOS
+ port - String of the port uri in ONOS
+ srcComponentLsit - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving taffic
+ shortDesc - String, Short description, used in reporting and file prefixes
+ longDesc - String, Longer description, used in logging
+ """
+ import time
+ try:
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDesc,
+ longDesc=longDesc )
+ main.step( "Port down" )
+ ctrl = main.Cluster.active( 0 ).CLI
+ portDown = ctrl.portstate( dpid=device, port=port, state="disable" )
+ portsJson = json.loads( ctrl.ports() )
+ for d in portsJson:
+ if d['device']['id'] == device:
+ for p in d['ports']:
+ if "(%s)" % port in p['port']:
+ adminState = p['isEnabled']
+ main.log.debug( adminState )
+ #TODO ASSERTS
+ main.log.info( "Sleeping 10 seconds" )
+ time.sleep(10)
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDesc,
+ longDesc=longDesc )
+ except Exception as e:
+ main.log.exception( "Error in linkDown" )
+
+ def linkUp( self, device, port, srcComponentList, dstComponent, shortDesc, longDesc ):
+ """"
+ High level function that handles an event including monitoring
+ Arguments:
+ device - String of the device uri in ONOS
+ port - String of the port uri in ONOS
+ srcComponentLsit - List containing src components, used for sending traffic
+ dstComponent - Component used for receiving taffic
+ shortDesc - String, Short description, used in reporting and file prefixes
+ longDesc - String, Longer description, used in logging
+ """
+ import time
+ try:
+ main.step( "Start Capturing" )
+ main.funcs.startCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDesc,
+ longDesc=longDesc )
+ main.step( "Port Up" )
+ ctrl = main.Cluster.active( 0 ).CLI
+ portUp = ctrl.portstate( dpid=device, port=port, state="enable" )
+ portsJson = json.loads( ctrl.ports() )
+ for d in portsJson:
+ if d['device']['id'] == device:
+ for p in d['ports']:
+ if "(%s)" % port in p['port']:
+ adminState = p['isEnabled']
+ main.log.debug( adminState )
+ #TODO ASSERTS
+ main.log.info( "Sleeping 10 seconds" )
+ time.sleep(10)
+ main.step( "Stop Capturing" )
+ main.funcs.stopCapturing( main,
+ srcComponentList,
+ dstComponent,
+ shortDesc=shortDesc,
+ longDesc=longDesc )
+ except Exception as e:
+ main.log.exception( "Error in linkUp" )
+
+ def switchDown( self ):
+ try:
+ pass
+ except Exception as e:
+ main.log.exception( "Error in switchDown" )
+
+ def switchUp( self ):
+ try:
+ pass
+ except Exception as e:
+ main.log.exception( "Error in switchUp" )
+
+ def onosDown( self ):
+ try:
+ pass
+ except Exception as e:
+ main.log.exception( "Error in onosDown" )
+
+ def analyzePcap( self, component, filePath, packetFilter, debug=False ):
+ try:
+ try:
+ output = ""
+ component.handle.sendline( "" )
+ while True:
+ component.handle.expect( component.prompt, timeout=1 )
+ output += component.handle.before + str( component.handle.after )
+ except pexpect.TIMEOUT:
+ main.log.debug( "%s: %s" % ( component.name, output ) )
+ except Exception as e:
+ main.log.exception( "Error in onosDown" )
+ lineRE = r'^\s*\d+\s+([0-9.]+)'
+ tsharkOptions = "-t dd -r %s -Y %s -T fields -e frame.number -e frame.time_delta -e ip.src -e ip.dst -e udp" % ( filePath, packetFilter )
+ component.handle.sendline( "sudo /usr/bin/tshark %s" % tsharkOptions )
+ i = component.handle.expect( [ "appears to be damaged or corrupt.", "Malformed Packet", component.prompt, pexpect.TIMEOUT ], timeout=60 )
+ if i != 2:
+ main.log.error( "Error Reading pcap file" )
+ component.handle.send( '\x03' ) # CTRL-C to end process
+ component.handle.expect( component.prompt )
+ main.log.debug( component.handle.before )
+ return 0
+ output = component.handle.before
+ deltas = []
+ for line in output.splitlines():
+ # Search for a packet in each line
+ # If match, save the delta time of the packet
+ m = re.search( lineRE, line )
+ if m:
+ if debug:
+ main.log.debug( repr( line ) )
+ main.log.info( m.groups() )
+ deltas.append( float( m.group(1) ) * 1000 )
+ else:
+ main.log.warn( repr( line ) )
+ if not deltas:
+ main.log.error( "No Packets found" )
+ return 0
+ # Print largest timestamp gap
+ deltas.sort()
+ if debug:
+ main.log.debug( deltas[ -10: ] ) # largest 10
+ main.log.info( "%s: Detected downtime (longest gap between packets): %s ms" % ( component.name, deltas[ -1 ] ) )
+ return deltas[ -1 ]
+ except Exception as e:
+ main.log.exception( "Error in analyzePcap" )
+
+ def dbWrite( self, main, filename ):
+ try:
+ dbFileName = "%s/%s" % ( main.logdir, filename )
+ dbfile = open( dbFileName, "w+" )
+ header = []
+ row = []
+ for eventName, results in main.downtimeResults.iteritems():
+ for measurementName, value in results.iteritems():
+ header.append( "'%s-%s'" % ( eventName, measurementName ) )
+ row.append( "'%s'" % value )
+ dbfile.write( ",".join( header ) + "\n" + ",".join( row ) + "\n" )
+ dbfile.close()
+ except IOError:
+ main.log.warn( "Error opening " + dbFileName + " to write results." )
+
+ def cleanup( self, main ):
+ # TODO: Do things like restore log levels here
+ run.cleanup( main )
+ self.dbWrite( main, "SRStaging-dbfile.csv")
+
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/dependencies/__init__.py