Merge "[SDFAB-504] Add UP4 encap, decap test"
diff --git a/TestON/bin/cleanup.sh b/TestON/bin/cleanup.sh
index 4810002..571b0c9 100755
--- a/TestON/bin/cleanup.sh
+++ b/TestON/bin/cleanup.sh
@@ -25,10 +25,11 @@
 
 # TODO: Add help to this file, and some more options?
 #       Maybe kill/uninstall ONOS?
-sudo kill -9 `ps -ef | grep "./cli.py" | grep -v grep | awk '{print $2}'`
-sudo kill -9 `ps -ef | grep "bin/teston" | grep -v grep | awk '{print $2}'`
-sudo kill -9 `ps -ef | grep "ssh -X" | grep -v grep | awk '{print $2}'`
-sudo kill -9 `ps ax | grep '[p]ython -m SimpleHTTPServer 8000' | awk '{print $1}'`
+sudo kill -9 `ps -ef | grep "./cli.py" | grep -v grep | awk '{print $2}'` &> /dev/null
+sudo kill -9 `ps -ef | grep "bin/teston" | grep -v grep | awk '{print $2}'` &> /dev/null
+sudo kill -9 `ps -ef | grep "/usr/bin/ssh" | grep -v grep | awk '{print $2}'` &> /dev/null
+sudo kill -9 `ps ax | grep '[p]ython -m SimpleHTTPServer 8000' | awk '{print $1}'` &> /dev/null
+sudo kill -9 `ps ax | grep 'kubectl' | awk '{print $2}'` &> /dev/null
 
 export user=${ONOS_USER:-$USER}
 if [[ -z "${OCN}" ]]; then
diff --git a/TestON/drivers/common/api/controller/trexclientdriver.py b/TestON/drivers/common/api/controller/trexclientdriver.py
new file mode 100644
index 0000000..29c8a1a
--- /dev/null
+++ b/TestON/drivers/common/api/controller/trexclientdriver.py
@@ -0,0 +1,581 @@
+"""
+Copyright 2021 Open Networking Foundation (ONF)
+
+Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
+the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
+or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
+
+"""
+import time
+import os
+import collections
+import numpy as np
+
+from drivers.common.api.controllerdriver import Controller
+from trex.stl.api import STLClient, STLStreamDstMAC_PKT
+from trex_stf_lib.trex_client import CTRexClient
+from trex_stl_lib.api import STLFlowLatencyStats, STLPktBuilder, STLStream, \
+    STLTXCont
+
+from socket import error as ConnectionRefusedError
+from distutils.util import strtobool
+
+TREX_FILES_DIR = "/tmp/trex_files/"
+
+LatencyStats = collections.namedtuple(
+    "LatencyStats",
+    [
+        "pg_id",
+        "jitter",
+        "average",
+        "total_max",
+        "total_min",
+        "last_max",
+        "histogram",
+        "dropped",
+        "out_of_order",
+        "duplicate",
+        "seq_too_high",
+        "seq_too_low",
+        "percentile_50",
+        "percentile_75",
+        "percentile_90",
+        "percentile_99",
+        "percentile_99_9",
+        "percentile_99_99",
+        "percentile_99_999",
+    ],
+)
+
+PortStats = collections.namedtuple(
+    "PortStats",
+    [
+        "tx_packets",
+        "rx_packets",
+        "tx_bytes",
+        "rx_bytes",
+        "tx_errors",
+        "rx_errors",
+        "tx_bps",
+        "tx_pps",
+        "tx_bps_L1",
+        "tx_util",
+        "rx_bps",
+        "rx_pps",
+        "rx_bps_L1",
+        "rx_util",
+    ],
+)
+
+FlowStats = collections.namedtuple(
+    "FlowStats",
+    [
+        "pg_id",
+        "tx_packets",
+        "rx_packets",
+        "tx_bytes",
+        "rx_bytes",
+    ],
+)
+
+
+class TrexClientDriver(Controller):
+    """
+    Implements a Trex Client Driver
+    """
+
+    def __init__(self):
+        self.trex_address = "localhost"
+        self.trex_config = None  # Relative path in dependencies of the test using this driver
+        self.force_restart = True
+        self.sofware_mode = False
+        self.setup_successful = False
+        self.stats = None
+        self.trex_client = None
+        self.trex_daemon_client = None
+        super(TrexClientDriver, self).__init__()
+
+    def connect(self, **connectargs):
+        try:
+            for key in connectargs:
+                vars(self)[key] = connectargs[key]
+            for key in self.options:
+                if key == "trex_address":
+                    self.trex_address = self.options[key]
+                elif key == "trex_config":
+                    self.trex_config = self.options[key]
+                elif key == "force_restart":
+                    self.force_restart = bool(strtobool(self.options[key]))
+                elif key == "software_mode":
+                    self.software_mode = bool(strtobool(self.options[key]))
+            self.name = self.options["name"]
+        except Exception as inst:
+            main.log.error("Uncaught exception: " + str(inst))
+            main.cleanAndExit()
+        return super(TrexClientDriver, self).connect()
+
+    def disconnect(self):
+        """
+        Called when Test is complete
+        """
+        self.disconnectTrexClient()
+        self.stopTrexServer()
+        return main.TRUE
+
+    def setupTrex(self, pathToTrexConfig):
+        """
+        Setup TRex server passing the TRex configuration.
+        :return: True if setup successful, False otherwise
+        """
+        main.log.debug(self.name + ": Setting up TRex server")
+        if self.software_mode:
+            trex_args = "--software --no-hw-flow-stat"
+        else:
+            trex_args = None
+        self.trex_daemon_client = CTRexClient(self.trex_address,
+                                              trex_args=trex_args)
+        success = self.__set_up_trex_server(
+            self.trex_daemon_client, self.trex_address,
+            pathToTrexConfig + self.trex_config,
+            self.force_restart
+        )
+        if not success:
+            main.log.error("Failed to set up TRex daemon!")
+            return False
+        self.setup_successful = True
+        return True
+
+    def connectTrexClient(self):
+        if not self.setup_successful:
+            main.log.error("Cannot connect TRex Client, first setup TRex")
+            return False
+        main.log.info("Connecting TRex Client")
+        self.trex_client = STLClient(server=self.trex_address)
+        self.trex_client.connect()
+        self.trex_client.acquire()
+        self.trex_client.reset()  # Resets configs from all ports
+        self.trex_client.clear_stats()  # Clear status from all ports
+        # Put all ports to promiscuous mode, otherwise they will drop all
+        # incoming packets if the destination mac is not the port mac address.
+        self.trex_client.set_port_attr(self.trex_client.get_all_ports(),
+                                       promiscuous=True)
+        # Reset the used sender ports
+        self.all_sender_port = set()
+        self.stats = None
+        return True
+
+    def disconnectTrexClient(self):
+        # Teardown TREX Client
+        if self.trex_client is not None:
+            main.log.info("Tearing down STLClient...")
+            self.trex_client.stop()
+            self.trex_client.release()
+            self.trex_client.disconnect()
+            self.trex_client = None
+            # Do not reset stats
+
+    def stopTrexServer(self):
+        if self.trex_daemon_client is not None:
+            self.trex_daemon_client.stop_trex()
+            self.trex_daemon_client = None
+
+    def addStream(self, pkt, trex_port, l1_bps=None, percentage=None,
+                  delay=0, flow_id=None, flow_stats=False):
+        """
+        :param pkt: Scapy packet, TRex will send copy of this packet
+        :param trex_port: Port number to send packet from, must match a port in the TRex config file
+        :param l1_bps: L1 Throughput generated by TRex (mutually exclusive with percentage)
+        :param percentage: Percentage usage of the selected port bandwidth (mutually exlusive with l1_bps)
+        :param flow_id: Flow ID, required when saving latency statistics
+        :param flow_stats: True to measure flow statistics (latency and packet), False otherwise, might require software mode
+        :return: True if the stream is create, false otherwise
+        """
+        if (percentage is None and l1_bps is None) or (
+                percentage is not None and l1_bps is not None):
+            main.log.error(
+                "Either percentage or l1_bps must be provided when creating a stream")
+            return False
+        main.log.debug("Creating flow stream")
+        main.log.debug(
+            "port: %d, l1_bps: %s, percentage: %s, delay: %d, flow_id:%s, flow_stats: %s" % (
+                trex_port, str(l1_bps), str(percentage), delay, str(flow_id),
+                str(flow_stats)))
+        main.log.debug(pkt.summary())
+        if flow_stats:
+            traffic_stream = self.__create_latency_stats_stream(
+                pkt,
+                pg_id=flow_id,
+                isg=delay,
+                percentage=percentage,
+                l1_bps=l1_bps)
+        else:
+            traffic_stream = self.__create_background_stream(
+                pkt,
+                percentage=percentage,
+                l1_bps=l1_bps)
+        self.trex_client.add_streams(traffic_stream, ports=trex_port)
+        self.all_sender_port.add(trex_port)
+        return True
+
+    def startAndWaitTraffic(self, duration=10):
+        """
+        Start generating traffic and wait traffic to be send
+        :param duration: Traffic generation duration
+        :return:
+        """
+        if not self.trex_client:
+            main.log.error(
+                "Cannot start traffic, first connect the TRex client")
+            return False
+        main.log.info("Start sending traffic for %d seconds" % duration)
+        self.trex_client.start(list(self.all_sender_port), mult="1",
+                               duration=duration)
+        main.log.info("Waiting until all traffic is sent..")
+        self.trex_client.wait_on_traffic(ports=list(self.all_sender_port),
+                                         rx_delay_ms=100)
+        main.log.info("...traffic sent!")
+        # Reset sender port so we can run other tests with the same TRex client
+        self.all_sender_port = set()
+        main.log.info("Getting stats")
+        self.stats = self.trex_client.get_stats()
+        main.log.info("GOT stats")
+
+    def getFlowStats(self, flow_id):
+        if self.stats is None:
+            main.log.error("No stats saved!")
+            return None
+        return TrexClientDriver.__get_flow_stats(flow_id, self.stats)
+
+    def logFlowStats(self, flow_id):
+        main.log.info("Statistics for flow {}: {}".format(
+            flow_id,
+            TrexClientDriver.__get_readable_flow_stats(
+                self.getFlowStats(flow_id))))
+
+    def getLatencyStats(self, flow_id):
+        if self.stats is None:
+            main.log.error("No stats saved!")
+            return None
+        return TrexClientDriver.__get_latency_stats(flow_id, self.stats)
+
+    def logLatencyStats(self, flow_id):
+        main.log.info("Latency statistics for flow {}: {}".format(
+            flow_id,
+            TrexClientDriver.__get_readable_latency_stats(
+                self.getLatencyStats(flow_id))))
+
+    def getPortStats(self, port_id):
+        if self.stats is None:
+            main.log.error("No stats saved!")
+            return None
+        return TrexClientDriver.__get_port_stats(port_id, self.stats)
+
+    def logPortStats(self, port_id):
+        if self.stats is None:
+            main.log.error("No stats saved!")
+            return None
+        main.log.info("Statistics for port {}: {}".format(
+            port_id, TrexClientDriver.__get_readable_port_stats(
+                self.stats.get(port_id))))
+
+    # From ptf/test/common/ptf_runner.py
+    def __set_up_trex_server(self, trex_daemon_client, trex_address,
+                             trex_config,
+                             force_restart):
+        try:
+            main.log.info("Pushing Trex config %s to the server" % trex_config)
+            if not trex_daemon_client.push_files(trex_config):
+                main.log.error("Unable to push %s to Trex server" % trex_config)
+                return False
+
+            if force_restart:
+                main.log.info("Restarting TRex")
+                trex_daemon_client.kill_all_trexes()
+                time.sleep(1)
+
+            if not trex_daemon_client.is_idle():
+                main.log.info("The Trex server process is running")
+                main.log.warn(
+                    "A Trex server process is still running, "
+                    + "use --force-restart to kill it if necessary."
+                )
+                return False
+
+            trex_config_file_on_server = TREX_FILES_DIR + os.path.basename(
+                trex_config)
+            trex_daemon_client.start_stateless(cfg=trex_config_file_on_server)
+        except ConnectionRefusedError:
+            main.log.error(
+                "Unable to connect to server %s.\n" + "Did you start the Trex daemon?" % trex_address)
+            return False
+
+        return True
+
+    def __create_latency_stats_stream(self, pkt, pg_id,
+                                      name=None,
+                                      l1_bps=None,
+                                      percentage=None,
+                                      isg=0):
+        assert (percentage is None and l1_bps is not None) or (
+                percentage is not None and l1_bps is None)
+        return STLStream(
+            name=name,
+            packet=STLPktBuilder(pkt=pkt),
+            mode=STLTXCont(bps_L1=l1_bps, percentage=percentage),
+            isg=isg,
+            flow_stats=STLFlowLatencyStats(pg_id=pg_id)
+        )
+
+    def __create_background_stream(self, pkt, name=None, percentage=None,
+                                   l1_bps=None):
+        assert (percentage is None and l1_bps is not None) or (
+                percentage is not None and l1_bps is None)
+        return STLStream(
+            name=name,
+            packet=STLPktBuilder(pkt=pkt),
+            mode=STLTXCont(bps_L1=l1_bps, percentage=percentage)
+        )
+
+    # Multiplier for data rates
+    K = 1000
+    M = 1000 * K
+    G = 1000 * M
+
+    @staticmethod
+    def __to_readable(src, unit="bps"):
+        """
+        Convert number to human readable string.
+        For example: 1,000,000 bps to 1Mbps. 1,000 bytes to 1KB
+
+        :parameters:
+            src : int
+                the original data
+            unit : str
+                the unit ('bps', 'pps', or 'bytes')
+        :returns:
+            A human readable string
+        """
+        if src < 1000:
+            return "{:.1f} {}".format(src, unit)
+        elif src < 1000000:
+            return "{:.1f} K{}".format(src / 1000, unit)
+        elif src < 1000000000:
+            return "{:.1f} M{}".format(src / 1000000, unit)
+        else:
+            return "{:.1f} G{}".format(src / 1000000000, unit)
+
+    @staticmethod
+    def __get_readable_port_stats(port_stats):
+        opackets = port_stats.get("opackets", 0)
+        ipackets = port_stats.get("ipackets", 0)
+        obytes = port_stats.get("obytes", 0)
+        ibytes = port_stats.get("ibytes", 0)
+        oerrors = port_stats.get("oerrors", 0)
+        ierrors = port_stats.get("ierrors", 0)
+        tx_bps = port_stats.get("tx_bps", 0)
+        tx_pps = port_stats.get("tx_pps", 0)
+        tx_bps_L1 = port_stats.get("tx_bps_L1", 0)
+        tx_util = port_stats.get("tx_util", 0)
+        rx_bps = port_stats.get("rx_bps", 0)
+        rx_pps = port_stats.get("rx_pps", 0)
+        rx_bps_L1 = port_stats.get("rx_bps_L1", 0)
+        rx_util = port_stats.get("rx_util", 0)
+        return """
+        Output packets: {}
+        Input packets: {}
+        Output bytes: {} ({})
+        Input bytes: {} ({})
+        Output errors: {}
+        Input errors: {}
+        TX bps: {} ({})
+        TX pps: {} ({})
+        L1 TX bps: {} ({})
+        TX util: {}
+        RX bps: {} ({})
+        RX pps: {} ({})
+        L1 RX bps: {} ({})
+        RX util: {}""".format(
+            opackets,
+            ipackets,
+            obytes,
+            TrexClientDriver.__to_readable(obytes, "Bytes"),
+            ibytes,
+            TrexClientDriver.__to_readable(ibytes, "Bytes"),
+            oerrors,
+            ierrors,
+            tx_bps,
+            TrexClientDriver.__to_readable(tx_bps),
+            tx_pps,
+            TrexClientDriver.__to_readable(tx_pps, "pps"),
+            tx_bps_L1,
+            TrexClientDriver.__to_readable(tx_bps_L1),
+            tx_util,
+            rx_bps,
+            TrexClientDriver.__to_readable(rx_bps),
+            rx_pps,
+            TrexClientDriver.__to_readable(rx_pps, "pps"),
+            rx_bps_L1,
+            TrexClientDriver.__to_readable(rx_bps_L1),
+            rx_util,
+        )
+
+    @staticmethod
+    def __get_port_stats(port, stats):
+        """
+        :param port: int
+        :param stats:
+        :return:
+        """
+        port_stats = stats.get(port)
+        return PortStats(
+            tx_packets=port_stats.get("opackets", 0),
+            rx_packets=port_stats.get("ipackets", 0),
+            tx_bytes=port_stats.get("obytes", 0),
+            rx_bytes=port_stats.get("ibytes", 0),
+            tx_errors=port_stats.get("oerrors", 0),
+            rx_errors=port_stats.get("ierrors", 0),
+            tx_bps=port_stats.get("tx_bps", 0),
+            tx_pps=port_stats.get("tx_pps", 0),
+            tx_bps_L1=port_stats.get("tx_bps_L1", 0),
+            tx_util=port_stats.get("tx_util", 0),
+            rx_bps=port_stats.get("rx_bps", 0),
+            rx_pps=port_stats.get("rx_pps", 0),
+            rx_bps_L1=port_stats.get("rx_bps_L1", 0),
+            rx_util=port_stats.get("rx_util", 0),
+        )
+
+    @staticmethod
+    def __get_latency_stats(pg_id, stats):
+        """
+        :param pg_id: int
+        :param stats:
+        :return:
+        """
+
+        lat_stats = stats["latency"].get(pg_id)
+        lat = lat_stats["latency"]
+        # Estimate latency percentiles from the histogram.
+        l = list(lat["histogram"].keys())
+        l.sort()
+        all_latencies = []
+        for sample in l:
+            range_start = sample
+            if range_start == 0:
+                range_end = 10
+            else:
+                range_end = range_start + pow(10, (len(str(range_start)) - 1))
+            val = lat["histogram"][sample]
+            # Assume whole the bucket experienced the range_end latency.
+            all_latencies += [range_end] * val
+        q = [50, 75, 90, 99, 99.9, 99.99, 99.999]
+        percentiles = np.percentile(all_latencies, q)
+
+        ret = LatencyStats(
+            pg_id=pg_id,
+            jitter=lat["jitter"],
+            average=lat["average"],
+            total_max=lat["total_max"],
+            total_min=lat["total_min"],
+            last_max=lat["last_max"],
+            histogram=lat["histogram"],
+            dropped=lat_stats["err_cntrs"]["dropped"],
+            out_of_order=lat_stats["err_cntrs"]["out_of_order"],
+            duplicate=lat_stats["err_cntrs"]["dup"],
+            seq_too_high=lat_stats["err_cntrs"]["seq_too_high"],
+            seq_too_low=lat_stats["err_cntrs"]["seq_too_low"],
+            percentile_50=percentiles[0],
+            percentile_75=percentiles[1],
+            percentile_90=percentiles[2],
+            percentile_99=percentiles[3],
+            percentile_99_9=percentiles[4],
+            percentile_99_99=percentiles[5],
+            percentile_99_999=percentiles[6],
+        )
+        return ret
+
+    @staticmethod
+    def __get_readable_latency_stats(stats):
+        """
+        :param stats: LatencyStats
+        :return:
+        """
+        histogram = ""
+        # need to listify in order to be able to sort them.
+        l = list(stats.histogram.keys())
+        l.sort()
+        for sample in l:
+            range_start = sample
+            if range_start == 0:
+                range_end = 10
+            else:
+                range_end = range_start + pow(10, (len(str(range_start)) - 1))
+            val = stats.histogram[sample]
+            histogram = (
+                    histogram
+                    + "\n        Packets with latency between {0:>5} us and {1:>5} us: {2:>10}".format(
+                range_start, range_end, val
+            )
+            )
+
+        return """
+        Latency info for pg_id {}
+        Dropped packets: {}
+        Out-of-order packets: {}
+        Sequence too high packets: {}
+        Sequence too low packets: {}
+        Maximum latency: {} us
+        Minimum latency: {} us
+        Maximum latency in last sampling period: {} us
+        Average latency: {} us
+        50th percentile latency: {} us
+        75th percentile latency: {} us
+        90th percentile latency: {} us
+        99th percentile latency: {} us
+        99.9th percentile latency: {} us
+        99.99th percentile latency: {} us
+        99.999th percentile latency: {} us
+        Jitter: {} us
+        Latency distribution histogram: {}
+        """.format(stats.pg_id, stats.dropped, stats.out_of_order,
+                   stats.seq_too_high, stats.seq_too_low, stats.total_max,
+                   stats.total_min, stats.last_max, stats.average,
+                   stats.percentile_50, stats.percentile_75,
+                   stats.percentile_90,
+                   stats.percentile_99, stats.percentile_99_9,
+                   stats.percentile_99_99,
+                   stats.percentile_99_999, stats.jitter, histogram)
+
+    @staticmethod
+    def __get_flow_stats(pg_id, stats):
+        """
+        :param pg_id: int
+        :param stats:
+        :return:
+        """
+        FlowStats = collections.namedtuple(
+            "FlowStats",
+            ["pg_id", "tx_packets", "rx_packets", "tx_bytes", "rx_bytes", ],
+        )
+        flow_stats = stats["flow_stats"].get(pg_id)
+        ret = FlowStats(
+            pg_id=pg_id,
+            tx_packets=flow_stats["tx_pkts"]["total"],
+            rx_packets=flow_stats["rx_pkts"]["total"],
+            tx_bytes=flow_stats["tx_bytes"]["total"],
+            rx_bytes=flow_stats["rx_bytes"]["total"],
+        )
+        return ret
+
+    @staticmethod
+    def __get_readable_flow_stats(stats):
+        """
+        :param stats: FlowStats
+        :return:
+        """
+        return """Flow info for pg_id {}
+        TX packets: {}
+        RX packets: {}
+        TX bytes: {}
+        RX bytes: {}""".format(stats.pg_id, stats.tx_packets,
+                               stats.rx_packets, stats.tx_bytes,
+                               stats.rx_bytes)