TestON/tests/USECASE/SegmentRouting/PMastership/PMastership.py

class PMastership:

    def __init__(self):
        self.default = ''

    def CASE1(self, main):
        main.case("PMastership Test")
        """
        Verify there are no pending flows and groups
        Get flows and group counts
        Verify that are not 0
        Get the master of leaf2 (look at the params file for the config)
        Verify that has the master
        Kill switch leaf2
        Set label on switch K8S node to prevent K8S to redeploy stratum
        Verify there are no pending flows and groups related to segment routing
        Verify that the master of leaf2 is still the same as before
        Wait for the switch to be up again
        Verify there are no pending flows and groups
        """
        try:
            from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
                Testcaselib as run
            from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import \
                SRStagingTest
            import time
        except ImportError as e:
            main.log.error("Import not found. Exiting the test")
            main.log.error(e)
            main.cleanAndExit()
        # Retrieves the params of the test
        n_switches = int(main.params["TOPO"]["switchNum"])
        switch_to_kill = main.params["PMastership"]["PMastership_dataplane_fail"]["switch_to_kill"]
        k8s_switch_node = main.params["PMastership"]["PMastership_dataplane_fail"]["k8s_switch_node"]
        k8s_label = main.params["PMastership"]["PMastership_dataplane_fail"]["k8s_label"]
        k8s_label_value_test = main.params["PMastership"]["PMastership_dataplane_fail"]["k8s_label_value_test"]
        k8s_label_value_normal = main.params["PMastership"]["PMastership_dataplane_fail"]["k8s_label_value_normal"]
        # Init the main components and variables
        run.initTest(main)
        main.log.info(main.Cluster.numCtrls)
        main.Cluster.setRunningNode(3)
        run.installOnos(main, skipPackage=True, cliSleep=5)
        onos_cli = main.Cluster.active(0).CLI
        kubectl = main.Cluster.active(0).Bench
        kubeconfig = main.Cluster.active(0).k8s.kubeConfig
        namespace = main.params['kubernetes']['namespace']

        main.step("Verify there are added flows")
        initial_flows_count = onos_cli.checkFlowCount()
        empty = main.TRUE if ( initial_flows_count == 0 ) else main.FALSE
        utilities.assert_equal(
            expect=main.FALSE,
            actual=empty,
            onpass="There are " + str(initial_flows_count) + " added flows",
            onfail="There are no added flows",
        )

        main.step("Verify there are added groups")
        initial_groups_count = onos_cli.checkGroupCount()
        empty = main.TRUE if ( initial_groups_count == 0 ) else main.FALSE
        utilities.assert_equal(
            expect=main.FALSE,
            actual=empty,
            onpass="There are " + str(initial_groups_count) + " added groups",
            onfail="There are no added groups",
        )

        no_pending_flows = utilities.retry(onos_cli.checkFlowsState,
                                           [False, None],
                                           kwargs={"isPENDING": False},
                                           attempts=20,
                                           getRetryingTime=True)

        main.step("Verify there are no pending flows")
        utilities.assert_equal(
            expect=main.TRUE,
            actual=no_pending_flows,
            onpass="There are no pending flows",
            onfail="There are pending flows",
        )

        no_pending_groups = utilities.retry(onos_cli.checkGroupsState,
                                            [False, None],
                                            kwargs={"isPENDING": False},
                                            attempts=20,
                                            getRetryingTime=True)

        main.step("Verify there are no pending groups")
        utilities.assert_equal(
            expect=main.TRUE,
            actual=no_pending_groups,
            onpass="There are no pending groups",
            onfail="There are pending groups",
        )

        main.step("Retrieving " + switch_to_kill + " master")
        initial_master = onos_cli.getMaster("device:" + k8s_switch_node)
        no_master = main.TRUE if ( initial_master is None ) else main.FALSE
        utilities.assert_equal(
            expect=main.FALSE,
            actual=no_master,
            onpass=initial_master + " is the master of " + switch_to_kill,
            onfail="There is no master for " + switch_to_kill,
        )

        main.step("Set label to switch k8s node and kill Stratum")
        # K8s node name correspond to the switch name in lowercase
        utilities.assert_equal(
            expect=main.TRUE,
            actual=kubectl.kubectlSetLabel(
                nodeName=k8s_switch_node,
                label=k8s_label,
                value=k8s_label_value_test,
                kubeconfig=kubeconfig,
                namespace=namespace,
            ),
            onpass="Label has been set correctly on node %s" % k8s_switch_node,
            onfail="Label has not been set on node %s" % k8s_switch_node
        )

        try:
            def checkNumberStratumPods(n_value):
                pods = kubectl.kubectlGetPodNames(
                    kubeconfig=kubeconfig,
                    namespace=namespace,
                    name="stratum"
                )
                main.log.info("PODS: " + str(pods))
                return n_value == len(pods) if pods is not main.FALSE else False
            # Execute the following in try/except/finally to be sure to restore the
            # k8s label even in case of unhandled exception.

            # Wait for stratum pod to be removed from the switch
            removed = utilities.retry(checkNumberStratumPods,
                                      False,
                                      args=[n_switches - 1],
                                      attempts=50)
            main.log.info("Stratum has been removed from the switch? %s" % removed)

            sleepTime = 20
            switch_component = getattr(main, switch_to_kill)
            main.log.info("Sleeping %s seconds for ONOS to react" % sleepTime)
            time.sleep(sleepTime)

            available = utilities.retry(SRStagingTest.switchIsConnected,
                                        True,
                                        args=[switch_component],
                                        attempts=300,
                                        getRetryingTime=True)
            main.log.info("Switch %s is available in ONOS? %s" % (
                switch_to_kill, available))
            utilities.assert_equal(
                expect=True,
                actual=not available and removed,
                onpass="Stratum was removed from switch k8s node",
                onfail="Stratum was not removed from switch k8s node"
            )

            main.step("Verify there are no segmentrouting flows after the failure")
            raw_flows = onos_cli.flows(device="device:" + k8s_switch_node)
            sr_flows = main.TRUE if "segmentrouting" in raw_flows else main.FALSE
            utilities.assert_equal(
                expect=main.FALSE,
                actual=sr_flows,
                onpass="There are no segmentrouting flows",
                onfail="There are segmentrouting flows",
            )

            main.step("Verify there are no segmentrouting groups after the failure")
            raw_groups = onos_cli.groups(device="device:" + k8s_switch_node)
            sr_groups = main.TRUE if "segmentrouting" in raw_groups else main.FALSE
            utilities.assert_equal(
                expect=main.FALSE,
                actual=sr_groups,
                onpass="There are no segmentrouting groups",
                onfail="There are segmentrouting groups",
            )

            main.step("Verify " + initial_master + " is still the master of " + switch_to_kill)
            after_master = onos_cli.getMaster("device:" + k8s_switch_node)
            no_master = main.TRUE if ( initial_master is None ) else main.FALSE
            utilities.assert_equal(
                expect=main.FALSE,
                actual=no_master,
                onpass=initial_master + " is the master of " + switch_to_kill,
                onfail="There is no master for " + switch_to_kill,
            )

            same_master = main.TRUE if ( initial_master == after_master ) else main.FALSE
            utilities.assert_equal(
                expect=main.TRUE,
                actual=same_master,
                onpass=initial_master + " is still the master of " + switch_to_kill,
                onfail="Master for " + switch_to_kill + " is " + after_master,
            )

        except Exception as e:
            main.log.error("Unhandled exception!")
            main.log.error(e)
        finally:
            utilities.assert_equal(
                expect=main.TRUE,
                actual=kubectl.kubectlSetLabel(
                    nodeName=k8s_switch_node,
                    label=k8s_label,
                    value=k8s_label_value_normal,
                    kubeconfig=kubeconfig,
                    namespace=namespace,
                ),
                onpass="Label has been set correctly on node %s" % k8s_switch_node,
                onfail="Label has not been set on node %s" % k8s_switch_node
            )

            # Wait for stratum pod to be re-deployed on the switch
            deployed = utilities.retry(checkNumberStratumPods,
                                       False,
                                       args=[n_switches],
                                       attempts=50)
            main.log.info("Stratum has been redeployed on the switch? %s" % deployed)

            # Wait switch to be back in ONOS
            available = utilities.retry(SRStagingTest.switchIsConnected,
                                        False,
                                        args=[switch_component],
                                        sleep=2,
                                        attempts=300,
                                        getRetryingTime=True)
            main.log.info("Switch %s is available in ONOS? %s" % (
                switch_to_kill, available))
            utilities.assert_equal(
                expect=True,
                actual=available and deployed,
                onpass="Switch is back available in ONOS and stratum has been redeployed",
                onfail="Switch is not available in ONOS, may influence subsequent tests!"
            )

        sleepTime = 10
        main.log.info("Sleeping %s seconds for ONOS to react and assure flows/groups are ADDED" % sleepTime)
        time.sleep(sleepTime)

        main.step("Verify there are added flows after reboot")
        after_flows_count = onos_cli.checkFlowCount()
        empty = main.TRUE if ( after_flows_count == 0 ) else main.FALSE
        utilities.assert_equal(
            expect=main.FALSE,
            actual=empty,
            onpass="There are " + str(after_flows_count) + " added flows",
            onfail="There are no added flows",
        )

        main.step("Verify there are added groups after reboot")
        after_groups_count = onos_cli.checkGroupCount()
        empty = main.TRUE if ( after_groups_count == 0 ) else main.FALSE
        utilities.assert_equal(
            expect=main.FALSE,
            actual=empty,
            onpass="There are " + str(after_groups_count) + " added groups",
            onfail="There are no added groups",
        )

        no_pending_flows = utilities.retry(onos_cli.checkFlowsState,
                                           [False, None],
                                           kwargs={"isPENDING": False},
                                           attempts=20,
                                           getRetryingTime=True)

        main.step("Verify there are no pending flows after reboot")
        utilities.assert_equal(
            expect=main.TRUE,
            actual=no_pending_flows,
            onpass="There are no pending flows",
            onfail="There are pending flows",
        )

        no_pending_groups = utilities.retry(onos_cli.checkGroupsState,
                                            [False, None],
                                            kwargs={"isPENDING": False},
                                            attempts=20,
                                            getRetryingTime=True)

        main.step("Verify there are no pending groups after reboot")
        utilities.assert_equal(
            expect=main.TRUE,
            actual=no_pending_groups,
            onpass="There are no pending groups",
            onfail="There are pending groups",
        )