core/net/src/test/java/org/onosproject/cluster/impl/MastershipManagerTest.java

/*
 * Copyright 2014-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.onosproject.cluster.impl;

import java.util.List;
import java.util.Set;
import java.util.function.Consumer;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.onlab.junit.TestUtils;
import org.onlab.packet.IpAddress;
import org.onosproject.cfg.ComponentConfigService;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.ControllerNode;
import org.onosproject.cluster.DefaultControllerNode;
import org.onosproject.cluster.NodeId;
import org.onosproject.common.event.impl.TestEventDispatcher;
import org.onosproject.mastership.MastershipInfo;
import org.onosproject.mastership.MastershipService;
import org.onosproject.mastership.MastershipStore;
import org.onosproject.mastership.MastershipTermService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.config.NetworkConfigServiceAdapter;
import org.onosproject.net.region.Region;
import org.onosproject.net.region.RegionId;
import org.onosproject.net.region.RegionStore;
import org.onosproject.net.region.impl.RegionManager;
import org.onosproject.store.cluster.StaticClusterService;
import org.onosproject.store.region.impl.DistributedRegionStore;
import org.onosproject.store.service.TestStorageService;
import org.onosproject.store.trivial.SimpleMastershipStore;

import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Futures;
import org.onosproject.upgrade.impl.UpgradeServiceAdapter;

import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.junit.Assert.*;
import static org.onosproject.net.MastershipRole.MASTER;
import static org.onosproject.net.MastershipRole.NONE;
import static org.onosproject.net.MastershipRole.STANDBY;
import static org.onosproject.net.NetTestTools.injectEventDispatcher;
import static org.onosproject.net.region.Region.Type.METRO;

/**
 * Test codifying the mastership service contracts.
 */
public class MastershipManagerTest {

    private static final NodeId NID_LOCAL = new NodeId("local");
    private static final NodeId NID_OTHER = new NodeId("foo");
    private static final IpAddress LOCALHOST = IpAddress.valueOf("127.0.0.1");
    private static final DeviceId DEV_MASTER = DeviceId.deviceId("of:1");
    private static final DeviceId DEV_OTHER = DeviceId.deviceId("of:2");

    private static final RegionId RID1 = RegionId.regionId("r1");
    private static final RegionId RID2 = RegionId.regionId("r2");
    private static final DeviceId DID1 = DeviceId.deviceId("foo:d1");
    private static final DeviceId DID2 = DeviceId.deviceId("foo:d2");
    private static final DeviceId DID3 = DeviceId.deviceId("foo:d3");
    private static final DeviceId DID4 = DeviceId.deviceId("foo:d4");
    private static final DeviceId DID5 = DeviceId.deviceId("foo:d5");
    private static final DeviceId DID6 = DeviceId.deviceId("foo:d6");
    private static final NodeId NID1 = NodeId.nodeId("n1");
    private static final NodeId NID2 = NodeId.nodeId("n2");
    private static final NodeId NID3 = NodeId.nodeId("n3");
    private static final NodeId NID4 = NodeId.nodeId("n4");
    private static final ControllerNode CNODE1 =
            new DefaultControllerNode(NID1, IpAddress.valueOf("127.0.1.1"));
    private static final ControllerNode CNODE2 =
            new DefaultControllerNode(NID2, IpAddress.valueOf("127.0.1.2"));
    private static final ControllerNode CNODE3 =
            new DefaultControllerNode(NID3, IpAddress.valueOf("127.0.1.3"));
    private static final ControllerNode CNODE4 =
            new DefaultControllerNode(NID4, IpAddress.valueOf("127.0.1.4"));


    private MastershipManager mgr;
    protected MastershipService service;
    private TestRegionManager regionManager;
    private RegionStore regionStore;
    private TestClusterService testClusterService;

    @Before
    public void setUp() throws Exception {
        mgr = new MastershipManager();
        service = mgr;
        injectEventDispatcher(mgr, new TestEventDispatcher());
        testClusterService = new TestClusterService();
        mgr.clusterService = testClusterService;
        mgr.upgradeService = new UpgradeServiceAdapter();
        mgr.store = new TestSimpleMastershipStore(mgr.clusterService);
        regionStore = new DistributedRegionStore();
        TestUtils.setField(regionStore, "storageService", new TestStorageService());
        TestUtils.callMethod(regionStore, "activate",
                             new Class<?>[] {});
        regionManager = new TestRegionManager();
        TestUtils.setField(regionManager, "store", regionStore);
        regionManager.activate();
        mgr.regionService = regionManager;

        ComponentConfigService mockConfigService =
                EasyMock.createMock(ComponentConfigService.class);
        expect(mockConfigService.getProperties(anyObject())).andReturn(ImmutableSet.of());
        mockConfigService.registerProperties(mgr.getClass());
        expectLastCall();
        mockConfigService.unregisterProperties(mgr.getClass(), false);
        expectLastCall();
        expect(mockConfigService.getProperties(anyObject())).andReturn(ImmutableSet.of());
        mgr.cfgService = mockConfigService;
        replay(mockConfigService);

        mgr.activate();
    }

    @After
    public void tearDown() {
        mgr.deactivate();
        mgr.clusterService = null;
        injectEventDispatcher(mgr, null);
        regionManager.deactivate();
        mgr.regionService = null;
        mgr.store = null;
    }

    @Test
    public void setRole() {
        mgr.setRole(NID_OTHER, DEV_MASTER, MASTER);
        assertEquals("wrong local role:", NONE, mgr.getLocalRole(DEV_MASTER));
        assertEquals("wrong obtained role:", STANDBY, Futures.getUnchecked(mgr.requestRoleFor(DEV_MASTER)));

        //set to master
        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
        assertEquals("wrong local role:", MASTER, mgr.getLocalRole(DEV_MASTER));
    }

    @Test
    public void relinquishMastership() {
        //no backups - should just turn to NONE for device.
        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
        assertEquals("wrong role:", MASTER, mgr.getLocalRole(DEV_MASTER));
        mgr.relinquishMastership(DEV_MASTER);
        assertNull("wrong master:", mgr.getMasterFor(DEV_OTHER));
        assertEquals("wrong role:", NONE, mgr.getLocalRole(DEV_MASTER));

        //not master, nothing should happen
        mgr.setRole(NID_LOCAL, DEV_OTHER, NONE);
        mgr.relinquishMastership(DEV_OTHER);
        assertNull("wrong role:", mgr.getMasterFor(DEV_OTHER));

        //provide NID_OTHER as backup and relinquish
        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
        assertEquals("wrong master:", NID_LOCAL, mgr.getMasterFor(DEV_MASTER));
        mgr.setRole(NID_OTHER, DEV_MASTER, STANDBY);
        mgr.relinquishMastership(DEV_MASTER);
        assertEquals("wrong master:", NID_OTHER, mgr.getMasterFor(DEV_MASTER));
    }

    @Test
    public void requestRoleFor() {
        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
        mgr.setRole(NID_OTHER, DEV_OTHER, MASTER);

        //local should be master for one but standby for other
        assertEquals("wrong role:", MASTER, Futures.getUnchecked(mgr.requestRoleFor(DEV_MASTER)));
        assertEquals("wrong role:", STANDBY, Futures.getUnchecked(mgr.requestRoleFor(DEV_OTHER)));
    }

    @Test
    public void getMasterFor() {
        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
        mgr.setRole(NID_OTHER, DEV_OTHER, MASTER);
        assertEquals("wrong master:", NID_LOCAL, mgr.getMasterFor(DEV_MASTER));
        assertEquals("wrong master:", NID_OTHER, mgr.getMasterFor(DEV_OTHER));

        //have NID_OTHER hand over DEV_OTHER to NID_LOCAL
        mgr.setRole(NID_LOCAL, DEV_OTHER, MASTER);
        assertEquals("wrong master:", NID_LOCAL, mgr.getMasterFor(DEV_OTHER));
    }

    @Test
    public void getDevicesOf() {
        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
        mgr.setRole(NID_LOCAL, DEV_OTHER, STANDBY);
        assertEquals("should be one device:", 1, mgr.getDevicesOf(NID_LOCAL).size());
        //hand both devices to NID_LOCAL
        mgr.setRole(NID_LOCAL, DEV_OTHER, MASTER);
        assertEquals("should be two devices:", 2, mgr.getDevicesOf(NID_LOCAL).size());
    }

    @Test
    public void termService() {
        MastershipTermService ts = mgr;

        //term = 1 for both
        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
        assertEquals("inconsistent term: ", 1, ts.getMastershipTerm(DEV_MASTER).termNumber());

        //hand devices to NID_LOCAL and back: term = 1 + 2
        mgr.setRole(NID_OTHER, DEV_MASTER, MASTER);
        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
        assertEquals("inconsistent terms: ", 3, ts.getMastershipTerm(DEV_MASTER).termNumber());
    }

    @Test
    public void balanceWithOrphans() {
        // Setup cluster of three nodes
        testClusterService.put(CNODE1, ControllerNode.State.ACTIVE);
        testClusterService.put(CNODE2, ControllerNode.State.INACTIVE);
        testClusterService.put(CNODE3, ControllerNode.State.ACTIVE);

        // Pre-assign some devices to each of the node
        // Leave some devices as orphans assigned to a downed node
        assignRoles(NID1, ImmutableSet.of(DID1, DID2, DID3, DID4));
        assignRoles(NID2, ImmutableSet.of(DID5));
        assignRoles(NID3, ImmutableSet.of(DID6));

        // Trigger load balancing
        mgr.balanceRoles();

        // Make sure we have a balanced load
        // Make sure that we no longer have any orphans
        assertEquals("incorrect balance for node 1", 3, mgr.getDevicesOf(NID1).size());
        assertEquals("incorrect balance for node 2", 0, mgr.getDevicesOf(NID2).size());
        assertEquals("incorrect balance for node 3", 3, mgr.getDevicesOf(NID3).size());
    }

    private void assignRoles(NodeId nid, Set<DeviceId> deviceIds) {
        Set<DeviceId> all = ImmutableSet.of(DID1, DID2, DID3, DID4, DID5, DID6);
        for (DeviceId did : all) {
            mgr.setRole(nid, did, deviceIds.contains(did) ? MASTER : STANDBY);
        }
    }

    @Test
    public void balanceWithRegion1() {
        //set up region - 2 sets of masters with 1 node in each
        Set<NodeId> masterSet1 = ImmutableSet.of(NID1);
        Set<NodeId> masterSet2 = ImmutableSet.of(NID2);
        List<Set<NodeId>> masters = ImmutableList.of(masterSet1, masterSet2);
        Region r = regionManager.createRegion(RID1, "R1", METRO, masters);
        regionManager.addDevices(RID1, ImmutableSet.of(DID1, DID2));
        Set<DeviceId> deviceIds = regionManager.getRegionDevices(RID1);
        assertEquals("incorrect device count", 2, deviceIds.size());

        testClusterService.put(CNODE1, ControllerNode.State.ACTIVE);
        testClusterService.put(CNODE2, ControllerNode.State.ACTIVE);

        //set master to non region nodes
        mgr.setRole(NID_LOCAL, DID1, MASTER);
        mgr.setRole(NID_LOCAL, DID2, MASTER);
        assertEquals("wrong local role:", MASTER, mgr.getLocalRole(DID1));
        assertEquals("wrong local role:", MASTER, mgr.getLocalRole(DID2));
        assertEquals("wrong master:", NID_LOCAL, mgr.getMasterFor(DID1));
        assertEquals("wrong master:", NID_LOCAL, mgr.getMasterFor(DID2));

        //do region balancing
        mgr.useRegionForBalanceRoles = true;
        mgr.balanceRoles();
        assertEquals("wrong master:", NID1, mgr.getMasterFor(DID1));
        assertEquals("wrong master:", NID1, mgr.getMasterFor(DID2));

        // make N1 inactive
        testClusterService.put(CNODE1, ControllerNode.State.INACTIVE);
        mgr.balanceRoles();
        assertEquals("wrong master:", NID2, mgr.getMasterFor(DID1));
        assertEquals("wrong master:", NID2, mgr.getMasterFor(DID2));

    }

    @Test
    public void balanceWithRegion2() {
        //set up region - 2 sets of masters with (3 nodes, 1 node)
        Set<NodeId> masterSet1 = ImmutableSet.of(NID1, NID3, NID4);
        Set<NodeId> masterSet2 = ImmutableSet.of(NID2);
        List<Set<NodeId>> masters = ImmutableList.of(masterSet1, masterSet2);
        Region r = regionManager.createRegion(RID1, "R1", METRO, masters);
        Set<DeviceId> deviceIdsOrig = ImmutableSet.of(DID1, DID2, DID3, DEV_OTHER);
        regionManager.addDevices(RID1, deviceIdsOrig);
        Set<DeviceId> deviceIds = regionManager.getRegionDevices(RID1);
        assertEquals("incorrect device count", deviceIdsOrig.size(), deviceIds.size());
        assertEquals("incorrect devices in region", deviceIdsOrig, deviceIds);

        testClusterService.put(CNODE1, ControllerNode.State.ACTIVE);
        testClusterService.put(CNODE2, ControllerNode.State.ACTIVE);
        testClusterService.put(CNODE3, ControllerNode.State.ACTIVE);
        testClusterService.put(CNODE4, ControllerNode.State.ACTIVE);

        //set master to non region nodes
        deviceIdsOrig.forEach(deviceId1 -> mgr.setRole(NID_LOCAL, deviceId1, MASTER));
        checkDeviceMasters(deviceIds, Sets.newHashSet(NID_LOCAL), deviceId ->
                assertEquals("wrong local role:", MASTER, mgr.getLocalRole(deviceId)));

        //do region balancing
        mgr.useRegionForBalanceRoles = true;
        mgr.balanceRoles();
        Set<NodeId> expectedMasters = Sets.newHashSet(NID1, NID3, NID4);
        checkDeviceMasters(deviceIds, expectedMasters);

        // make N1 inactive
        testClusterService.put(CNODE1, ControllerNode.State.INACTIVE);
        expectedMasters.remove(NID1);
        mgr.balanceRoles();
        checkDeviceMasters(deviceIds, expectedMasters);

        // make N4 inactive
        testClusterService.put(CNODE4, ControllerNode.State.INACTIVE);
        expectedMasters.remove(NID4);
        mgr.balanceRoles();
        checkDeviceMasters(deviceIds, expectedMasters);

        // make N3 inactive
        testClusterService.put(CNODE3, ControllerNode.State.INACTIVE);
        expectedMasters = Sets.newHashSet(NID2);
        mgr.balanceRoles();
        checkDeviceMasters(deviceIds, expectedMasters);

        // make N3 active
        testClusterService.put(CNODE3, ControllerNode.State.ACTIVE);
        expectedMasters = Sets.newHashSet(NID3);
        mgr.balanceRoles();
        checkDeviceMasters(deviceIds, expectedMasters);

        // make N4 active
        testClusterService.put(CNODE4, ControllerNode.State.ACTIVE);
        expectedMasters.add(NID4);
        mgr.balanceRoles();
        checkDeviceMasters(deviceIds, expectedMasters);

        // make N1 active
        testClusterService.put(CNODE1, ControllerNode.State.ACTIVE);
        expectedMasters.add(NID1);
        mgr.balanceRoles();
        checkDeviceMasters(deviceIds, expectedMasters);
    }

    @Test
    public void demote() {
        mgr.setRole(NID1, DID1, MASTER);
        mgr.setRole(NID2, DID1, STANDBY);
        mgr.setRole(NID3, DID1, STANDBY);
        List<NodeId> stdbys = Lists.newArrayList(NID2, NID3);
        MastershipInfo mastershipInfo = mgr.getMastershipFor(DID1);
        assertTrue(mastershipInfo.master().isPresent());
        assertEquals("wrong role", NID1, mastershipInfo.master().get());
        assertEquals("wrong backups", stdbys, mastershipInfo.backups());
        // No effect, it is the master
        mgr.demote(NID1, DID1);
        assertEquals("wrong role", NID1, mastershipInfo.master().get());
        assertEquals("wrong backups", stdbys, mastershipInfo.backups());
        // No effect, it is not part of the mastership
        mgr.demote(NID4, DID1);
        assertEquals("wrong role", NID1, mastershipInfo.master().get());
        assertEquals("wrong backups", stdbys, mastershipInfo.backups());
        // Demote N2
        mgr.demote(NID2, DID1);
        stdbys = Lists.newArrayList(NID3, NID2);
        mastershipInfo = mgr.getMastershipFor(DID1);
        assertEquals("wrong role", NID1, mastershipInfo.master().get());
        assertEquals("wrong backups", stdbys, mastershipInfo.backups());
    }

    private void checkDeviceMasters(Set<DeviceId> deviceIds, Set<NodeId> expectedMasters) {
        checkDeviceMasters(deviceIds, expectedMasters, null);
    }

    private void checkDeviceMasters(Set<DeviceId> deviceIds, Set<NodeId> expectedMasters,
                                     Consumer<DeviceId> checkRole) {
        // each device's master must be contained in the list of expectedMasters
        deviceIds.forEach(deviceId -> {
            assertTrue("wrong master:", expectedMasters.contains(mgr.getMasterFor(deviceId)));
            if (checkRole != null) {
                checkRole.accept(deviceId);
            }
        });
        // each node in expectedMasters must have approximately the same number of devices
        if (expectedMasters.size() > 1) {
            int minValue = Integer.MAX_VALUE;
            int maxDevices = -1;
            for (NodeId nodeId: expectedMasters) {
                int numDevicesManagedByNode = mgr.getDevicesOf(nodeId).size();
                if (numDevicesManagedByNode < minValue) {
                    minValue = numDevicesManagedByNode;
                }
                if (numDevicesManagedByNode > maxDevices) {
                    maxDevices = numDevicesManagedByNode;
                }
                assertTrue("not balanced:", maxDevices - minValue <= 1);
            }
        }
    }

    private final class TestClusterService extends StaticClusterService {

        ControllerNode local = new DefaultControllerNode(NID_LOCAL, LOCALHOST);

        @Override
        public ControllerNode getLocalNode() {
            return local;
        }

        public void put(ControllerNode cn, ControllerNode.State state) {
            nodes.put(cn.id(), cn);
            nodeStates.put(cn.id(), state);
        }
    }

    private final class TestSimpleMastershipStore extends SimpleMastershipStore
            implements MastershipStore {

        public TestSimpleMastershipStore(ClusterService clusterService) {
            super.clusterService = clusterService;
        }
    }

    private class TestRegionManager extends RegionManager {
        TestRegionManager() {
            eventDispatcher = new TestEventDispatcher();
            networkConfigService = new NetworkConfigServiceAdapter();
        }
    }
}