Add Stratum-bmv2 to Trellis SRBridging

TODO:
 - Refactor so that we can use with other SR Tests
 - Add creating docker image

Change-Id: Ib44418579e6f0eb4a862c36077459ed4dd389f8e
(cherry picked from commit 9bc3fb67167255b8962f4c1e4dc05d7fac10656b)
diff --git a/TestON/bin/cleanup.sh b/TestON/bin/cleanup.sh
index 23b288a..4810002 100755
--- a/TestON/bin/cleanup.sh
+++ b/TestON/bin/cleanup.sh
@@ -58,4 +58,6 @@
         ssh sdn@$i "sudo ip6tables -F"
         ssh sdn@$i "sudo ip6tables-restore < /etc/iptables/rules.v6"
     done
+elif [ "$1" = "-d" ]; then
+    docker stop $(docker ps -aq)
 fi
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index e67e4e1..55a106f 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -62,6 +62,7 @@
         self.bashPrompt = "\$"
         self.scapyPrompt = ">>>"
         self.graph = Graph()
+        self.sudoRequired = True
 
     def connect( self, **connectargs ):
         """
@@ -75,7 +76,8 @@
             for key in self.options:
                 if key == "home":
                     self.home = self.options[ 'home' ]
-                    break
+                elif key == "sudo_required":
+                    self.sudoRequired = False if self.options[ key ] == "false" else True
             if self.home is None or self.home == "":
                 self.home = "~/mininet"
 
@@ -143,7 +145,10 @@
                 # make sure old networks are cleaned up
                 main.log.info( self.name +
                                ": Clearing any residual state or processes" )
-                self.handle.sendline( "sudo mn -c" )
+                cmd = "mn -c"
+                if self.sudoRequired:
+                    cmd = "sudo " + cmd
+                self.handle.sendline( cmd )
                 i = self.handle.expect( [ 'password\sfor\s',
                                           'Cleanup\scomplete',
                                           pexpect.EOF,
@@ -166,7 +171,10 @@
                     main.log.error( self.name + ": Something while cleaning " +
                                     "Mininet took too long... " )
                 # Craft the string to start mininet
-                cmdString = "sudo "
+                if self.sudoRequired:
+                    cmdString = "sudo "
+                else:
+                    cmdString = ""
                 if not mnCmd:
                     if topoFile is None or topoFile == '':  # If no file is given
                         main.log.info( self.name + ": building fresh Mininet" )
@@ -185,7 +193,9 @@
                         main.log.info(
                             "Starting Mininet from topo file " +
                             topoFile )
-                        cmdString += "-E python " + topoFile + " "
+                        if self.sudoRequired:
+                            cmdString += "-E "
+                        cmdString += "python " + topoFile + " "
                         if args is None:
                             args = ''
                             # TODO: allow use of args from .topo file?
@@ -2281,7 +2291,10 @@
                                              prompt=self.prompt,
                                              timeout=exitTimeout )
                     main.log.info( self.name + ": Stopped\nTime Took : " + str( time.time() - startTime ) )
-                    self.handle.sendline( "sudo mn -c" )
+                    cmd = "mn -c"
+                    if self.sudoRequired:
+                        cmd = "sudo " + cmd
+                    self.handle.sendline( cmd )
                     response = main.TRUE
 
                 elif i == 1:
@@ -2296,15 +2309,20 @@
 
                 self.handle.sendline( "" )
                 self.handle.expect( self.prompt )
-                self.handle.sendline( "sudo killall -9 dhclient dhcpd zebra bgpd" )
+                cmd = "killall -9 dhclient dhcpd zebra bgpd"
+                if self.sudoRequired:
+                    cmd = "sudo " + cmd
+                self.handle.sendline( cmd )
+                self.handle.expect( self.prompt )
 
                 if fileName:
                     self.handle.sendline( "" )
                     self.handle.expect( self.prompt )
-                    self.handle.sendline(
-                        "sudo kill -9 \`ps -ef | grep \"" +
-                        fileName +
-                        "\" | grep -v grep | awk '{print $2}'\`" )
+                    cmd = "kill -9 \`ps -ef | grep \"" + fileName + "\" | grep -v grep | awk '{print $2}'\`"
+                    if self.sudoRequired:
+                        cmd = "sudo " + cmd
+                    self.handle.sendline( cmd )
+                    self.handle.expect( self.prompt )
             except pexpect.TIMEOUT:
                 main.log.error( self.name + ": TIMEOUT exception found" )
                 main.log.error( self.name + ":     " + self.handle.before )
@@ -2660,14 +2678,14 @@
         try:
             self.handle.sendline( "" )
             self.handle.expect( "mininet>" )
+            if self.sudoRequired:
+                sudoStr = "sudo "
+            else:
+                sudoStr = ""
             self.handle.sendline(
-                "sh sudo tcpdump -n -i " +
-                intf +
-                " " +
-                port +
-                " -w " +
-                filename.strip() +
-                "  &" )
+                "sh " + sudoStr + "tcpdump -n -i " +
+                intf + " " + port + " -w " +
+                filename.strip() + "  &" )
             self.handle.sendline( "" )
             i = self.handle.expect( [ 'No\ssuch\device',
                                       'listening\son',
@@ -2712,7 +2730,11 @@
         """
             pkills tcpdump"""
         try:
-            self.handle.sendline( "sh sudo pkill tcpdump" )
+            if self.sudoRequired:
+                sudoStr = "sudo "
+            else:
+                sudoStr = ""
+            self.handle.sendline( "sh " + sudoStr + " pkill tcpdump" )
             self.handle.expect( "mininet>" )
             self.handle.sendline( "" )
             self.handle.expect( "mininet>" )
@@ -2822,7 +2844,7 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
 
-    def getSwitches( self, verbose=False, updateTimeout=1000, excludeNodes=[] ):
+    def getSwitches( self, verbose=False, updateTimeout=1000, excludeNodes=[], switchRegex=[] ):
         """
         Read switches from Mininet.
 
@@ -2841,7 +2863,10 @@
         # <OVSSwitchNS s1: lo:127.0.0.1,s1-eth1:None,s1-eth2:None,s1-eth3:None pid=22550>
         # <OVSBridge s1: lo:127.0.0.1,s1-eth1:None,s1-eth2:None pid=26830>
         # <UserSwitch s1: lo:127.0.0.1,s1-eth1:None,s1-eth2:None pid=14737>
-        switchClasses = r"(OVSSwitch)|(OVSBridge)|(OVSSwitchNS)|(IVSSwitch)|(LinuxBridge)|(UserSwitch)"
+        if not switchRegex:
+            switchClasses = r"(OVSSwitch)|(OVSBridge)|(OVSSwitchNS)|(IVSSwitch)|(LinuxBridge)|(UserSwitch)"
+        else:
+            switchClasses = switchRegex
         try:
             swRE = r"<(?P<class>" + switchClasses + r")" +\
                    r"(?P<options>\{.*\})?\s" +\
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 78c4531..10a015d 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -437,6 +437,7 @@
 
             response = self.handle.before
             if re.search( "Error", response ):
+                main.log.debug( response )
                 return main.FALSE
             return main.TRUE
         except pexpect.TIMEOUT:
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/README.md b/TestON/tests/USECASE/SegmentRouting/SRBridging/README.md
index 7409f58..5b86f4c 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/README.md
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/README.md
@@ -10,6 +10,12 @@
  - Trellis leaf-spine fabric: please visit following URL to set up Trellis leaf-spine fabric
  https://github.com/opennetworkinglab/routing/tree/master/trellis
  - ONOS_APPS=drivers,openflow,segmentrouting,fpm,netcfghostprovider
+ - For the stratum bmv2 switch testing, the minient node requires docker. The docker image
+ can be built using dependencies/Dockerfile.
+ - The treliis mininet topology file requires the stratum.py file which can be found in the
+ stratum repo (github.com/stratum/stratum) at stratum/tools/mininet/stratum.py.
+ The stratum docker has this file already installed, but you may want to add this
+ to the mininet/custom folder as well.
 
 <h3>Topologies</h3>
 - 0x1 single ToR
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params
index 2d50ac0..97d83a1 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params
@@ -16,6 +16,8 @@
         <useCommonConf>False</useCommonConf>
         <useCommonTopo>True</useCommonTopo>
         <useBmv2>False</useBmv2>
+        <bmv2SwitchType>bmv2</bmv2SwitchType>
+        <stratumRoot>~/stratum</stratumRoot>
         <topology>trellis_fabric.py</topology>
         <lib>routinglib.py,trellislib.py</lib>
         <trellisOar>/home/sdn/segmentrouting-oar-3.0.0-SNAPSHOT.oar</trellisOar>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.stratum b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.stratum
new file mode 100644
index 0000000..e1e7395
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.params.stratum
@@ -0,0 +1,56 @@
+<PARAMS>
+    # Sample testcase for physical network: 7
+    <testcases>5,6,7,8,15,16,17,18,25,26,27,28,35,36,37,38,45,46,47,48,55,56,57,58,65,66,67,68,75,76,77,78</testcases>
+
+    <GRAPH>
+        <nodeCluster>Fabric</nodeCluster>
+        <builds>20</builds>
+    </GRAPH>
+
+    <SCALE>
+        <size>3</size>
+        <max>3</max>
+    </SCALE>
+
+    <DEPENDENCY>
+        <useCommonConf>False</useCommonConf>
+        <useCommonTopo>True</useCommonTopo>
+        <useBmv2>True</useBmv2>
+        <bmv2SwitchType>stratum</bmv2SwitchType>
+        <stratumRoot>~/stratum</stratumRoot>
+        <topology>trellis_fabric.py</topology>
+        <lib>routinglib.py,trellislib.py</lib>
+    </DEPENDENCY>
+
+    <MN_DOCKER>
+        <args>--privileged --net host --rm -v topo:/topo -v ~/mininet/custom:/root/mininet/custom -v /var/run/openvswitch/:/var/run/openvswitch/ -v /tmp/mn-stratum:/tmp --hostname mn-stratum -v /etc/network/interfaces:/etc/network/interfaces -it -d</args>
+        <name>trellis_mininet</name>
+    </MN_DOCKER>
+
+    <ENV>
+        <cellName>productionCell</cellName>
+        <cellApps>drivers,segmentrouting,openflow,fpm,netcfghostprovider,drivers.bmv2,pipelines.fabric</cellApps>
+    </ENV>
+
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+
+    <CTRL>
+        <port>6653</port>
+    </CTRL>
+
+    <timers>
+        <LinkDiscovery>12</LinkDiscovery>
+        <SwitchDiscovery>12</SwitchDiscovery>
+    </timers>
+
+    <SLEEP>
+        <startup>10</startup>
+    </SLEEP>
+
+    <ALARM>
+        <minPassPercent>100</minPassPercent>
+    </ALARM>
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py b/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
index af2848f..3368950 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
@@ -62,6 +62,31 @@
         run.loadJson( main )
         run.loadChart( main )
         if hasattr( main, 'Mininet1' ):
+            if 'MN_DOCKER' in main.params and main.params['MN_DOCKER']['args']:
+                main.log.info( "Creating Mininet Docker" )
+                main.Mininet1.dockerPrompt = '#'
+                # Start docker container
+                handle = main.Mininet1.handle
+                handle.sendline( "docker run %s %s" % ( main.params[ 'MN_DOCKER' ][ 'args' ], main.params[ 'MN_DOCKER' ][ 'name' ] ) )
+                handle.expect( main.Mininet1.bashPrompt )
+                output = handle.before + handle.after
+                main.log.debug( repr(output) )
+                startStr = main.params[ 'MN_DOCKER' ][ 'name' ]
+                endStr = main.Mininet1.user_name
+                start = output.find( startStr ) + len( startStr )
+                end = output.find( endStr )
+                containerId = output[start:end].strip()
+                main.log.debug( repr( containerId ) )
+
+                handle = main.Mininet1.handle
+                handle.sendline( "docker attach %s " % containerId )  # Strip?
+                handle.expect( main.Mininet1.dockerPrompt )
+                main.log.debug( handle.before + handle.after )
+                # We should be good to go
+                main.Mininet1.prompt = main.Mininet1.dockerPrompt
+                main.Mininet1.sudoRequired = False
+
+
             # Run the test with Mininet
             mininet_args = ' --spine=%d --leaf=%d' % ( self.topo[ topology ][ 0 ], self.topo[ topology ][ 1 ] )
             if self.topo[ topology ][ 2 ]:
@@ -69,10 +94,24 @@
             if len( vlan ) > 0 :
                 mininet_args += ' --vlan=%s' % ( ','.join( ['%d' % vlanId for vlanId in vlan ] ) )
             if main.useBmv2:
-                mininet_args += ' --switch bmv2'
-                main.log.info( "Using BMv2 switch" )
+                switchType = main.params[ 'DEPENDENCY' ].get( 'bmv2SwitchType', 'stratum' )
+                mininet_args += ' --switch %s' % switchType
+                main.log.info( "Using %s switch" % switchType )
 
             run.startMininet( main, 'trellis_fabric.py', args=mininet_args )
+
+            if main.useBmv2:
+                filename = "onos-netcfg.json"
+                for switch in main.Mininet1.getSwitches(switchRegex=r"(StratumBmv2Switch)|(Bmv2Switch)").keys():
+                    path = "/tmp/mn-stratum/%s/" % switch
+                    main.ONOSbench1.handle.sendline( "sudo sed -i 's/localhost/%s/g' %s%s" % ( main.Mininet1.ip_address, path, filename ) )
+                    main.ONOSbench1.handle.expect( main.ONOSbench1.prompt )
+                    main.log.debug( main.ONOSbench1.handle.before + main.ONOSbench1.handle.after )
+                    main.ONOSbench1.handle.sendline( "sudo sed -i 's/device:%s/device:bmv2:%s/g' %s%s" % ( switch, switch, path, filename ) )
+                    main.ONOSbench1.handle.expect( main.ONOSbench1.prompt )
+                    main.log.debug( main.ONOSbench1.handle.before + main.ONOSbench1.handle.after )
+                    main.ONOSbench1.onosNetCfg( main.ONOSserver1.ip_address, path, filename )
+
         else:
             # Run the test with physical devices
             run.connectToPhysicalNetwork( main, self.switchNames[ topology ] )
@@ -87,3 +126,22 @@
         run.pingAll( main )
 
         run.cleanup( main )
+        if 'MN_DOCKER' in main.params and main.params['MN_DOCKER']['args']:
+                main.log.info( "Deleting Mininet Docker" )
+
+                # Detach from container
+                handle = main.Mininet1.handle
+                try:
+                    handle.sendline( "exit" )  # ctrl-p ctrk-q  to detach from container
+                    import time
+                    time.sleep(5)
+                    handle.expect( main.Mininet1.dockerPrompt )
+                    main.log.debug( handle.before + handle.after )
+                    main.Mininet1.prompt = main.Mininet1.bashPrompt
+                    handle.expect( main.Mininet1.prompt )
+                    main.log.debug( handle.before + handle.after )
+                    main.Mininet1.sudoRequired = True
+                except Exception as e:
+                    main.log.error( e )
+
+                # We should be good to go
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Dockerfile b/TestON/tests/USECASE/SegmentRouting/dependencies/Dockerfile
new file mode 100644
index 0000000..1843055
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Dockerfile
@@ -0,0 +1,6 @@
+FROM opennetworking/mn-stratum
+
+RUN install_packages python-pip openvswitch-switch
+#RUN apt-get update && apt-get install -y python-pip
+RUN pip install ipaddress
+ENTRYPOINT bash
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 22988d8..efd8868 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -76,6 +76,7 @@
             main.topologyLib = main.params[ 'DEPENDENCY' ][ 'lib' ] if 'lib' in main.params[ 'DEPENDENCY' ] else None
             main.topologyConf = main.params[ 'DEPENDENCY' ][ 'conf' ] if 'conf' in main.params[ 'DEPENDENCY' ] else None
             main.bmv2 = "bmv2.py"
+            main.stratumRoot = main.params[ 'DEPENDENCY'][ 'stratumRoot'] if 'stratumRoot' in main.params[ 'DEPENDENCY' ] else None
             main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
             main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
             main.trellisOar = main.params[ 'DEPENDENCY' ][ 'trellisOar' ]
@@ -141,9 +142,21 @@
             ctrl.CLI.logSet( "DEBUG", "org.onosproject.routeservice.impl" )
             ctrl.CLI.logSet( "DEBUG", "org.onosproject.routeservice.store" )
             ctrl.CLI.logSet( "DEBUG", "org.onosproject.routing.fpm" )
+            ctrl.CLI.logSet( "DEBUG", "org.onosproject.fpm" )
             ctrl.CLI.logSet( "TRACE", "org.onosproject.events" )
             ctrl.CLI.logSet( "DEBUG", "org.onosproject.mcast" )
 
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.p4runtime" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.protocols.p4runtime" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.drivers.p4runtime" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.protocols.grpc" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.protocols.gnmi" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.protocols.gnoi" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.drivers.gnoi" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.drivers.gmni" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.drivers.stratum" )
+            ctrl.CLI.logSet( "TRACE", "org.onosproject.bmv2" )
+
     @staticmethod
     def loadCount( main ):
         with open("%s/count/%s.count" % (main.configPath, main.cfgName)) as count:
@@ -227,9 +240,14 @@
                                  actual=stepResult,
                                  onpass="Successfully copied topo files",
                                  onfail="Failed to copy topo files" )
+        if main.stratumRoot:
+            main.Mininet1.handle.sendline( "export STRATUM_ROOT=" + str( main.stratumRoot ) )
+            main.Mininet1.handle.expect( main.Mininet1.prompt )
         main.step( "Starting Mininet Topology" )
         arg = "--onos-ip=%s %s" % (",".join([ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args)
         main.topology = topology
+        #switchType = " --switch=stratum"
+        #arg += switchType
         topoResult = main.Mininet1.startNet(
                 topoFile=main.Mininet1.home + "custom/" + main.topology, args=arg )
         stepResult = topoResult
@@ -865,6 +883,8 @@
         """
         from tests.dependencies.utils import Utils
         main.utils = Utils()
+        for ctrl in main.Cluster.active():
+            ctrl.CLI.log( "\"Ending Test - Shutting down ONOS and Network\"", level="INFO" )
         # Clean up scapy hosts
         if hasattr( main, "scapyHosts" ):
             scapyResult = main.TRUE
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/trellis_fabric.py b/TestON/tests/USECASE/SegmentRouting/dependencies/trellis_fabric.py
index e02a675..8668f83 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/trellis_fabric.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/trellis_fabric.py
@@ -17,6 +17,7 @@
 from functools import partial
 
 from bmv2 import ONOSBmv2Switch
+from stratum import StratumBmv2Switch
 
 # Parse command line options and dump results
 def parseOptions():
@@ -46,7 +47,7 @@
     parser.add_option( '--remote-dhcp-server', action="store_true", dest='remoteServer', default=False,
                        help='Connect DHCP server indirectly (via gateway) if True' )
     parser.add_option( '--switch', dest='switch', type='str', default='ovs',
-                       help='Switch type: ovs, bmv2 (with fabric.p4)' )
+                       help='Switch type: ovs, bmv2 (with fabric.p4), stratum' )
     ( options, args ) = parser.parse_args()
     return options, args
 
@@ -58,7 +59,8 @@
 
 SWITCH_TO_PARAMS_DICT = {
     "ovs": dict(cls=OVSSwitch),
-    "bmv2": dict(cls=ONOSBmv2Switch, pipeconf=FABRIC_PIPECONF)
+    "bmv2": dict(cls=ONOSBmv2Switch, pipeconf=FABRIC_PIPECONF),
+    "stratum": dict(cls=StratumBmv2Switch, pipeconf=FABRIC_PIPECONF, loglevel='debug')
 }
 if opts.switch not in SWITCH_TO_PARAMS_DICT:
     raise Exception("Unknown switch type '%s'" % opts.switch)