[Goldeneye] ONOS-4017: Mastership service considers Region information when determining mastership.
Change-Id: I6c79239f2e071d865bf04e4d9d790ca9b2d04694
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/MastershipManager.java b/core/net/src/main/java/org/onosproject/cluster/impl/MastershipManager.java
index e746d92..2a30108 100644
--- a/core/net/src/main/java/org/onosproject/cluster/impl/MastershipManager.java
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/MastershipManager.java
@@ -18,6 +18,7 @@
import com.codahale.metrics.Timer;
import com.codahale.metrics.Timer.Context;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Futures;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
@@ -42,9 +43,13 @@
import org.onosproject.mastership.MastershipTermService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.MastershipRole;
+import org.onosproject.net.region.Region;
+import org.onosproject.net.region.RegionService;
import org.slf4j.Logger;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -89,8 +94,12 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected MetricsService metricsService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected RegionService regionService;
+
private NodeId localNodeId;
private Timer requestRoleTimer;
+ public boolean useRegionForBalanceRoles;
@Activate
public void activate() {
@@ -212,6 +221,28 @@
}
}
+ if (useRegionForBalanceRoles && balanceRolesUsingRegions(controllerDevices)) {
+ return;
+ }
+
+ // Now re-balance the buckets until they are roughly even.
+ List<CompletableFuture<Void>> balanceBucketsFutures = balanceControllerNodes(controllerDevices, deviceCount);
+
+ CompletableFuture<Void> balanceRolesFuture = CompletableFuture.allOf(
+ balanceBucketsFutures.toArray(new CompletableFuture[balanceBucketsFutures.size()]));
+
+ Futures.getUnchecked(balanceRolesFuture);
+ }
+
+ /**
+ * Balances the nodes specified in controllerDevices.
+ *
+ * @param controllerDevices controller nodes to devices map
+ * @param deviceCount number of devices mastered by controller nodes
+ * @return list of setRole futures for "moved" devices
+ */
+ private List<CompletableFuture<Void>> balanceControllerNodes(
+ Map<ControllerNode, Set<DeviceId>> controllerDevices, int deviceCount) {
// Now re-balance the buckets until they are roughly even.
List<CompletableFuture<Void>> balanceBucketsFutures = Lists.newLinkedList();
int rounds = controllerDevices.keySet().size();
@@ -221,12 +252,16 @@
ControllerNode largest = findBucket(false, controllerDevices);
balanceBucketsFutures.add(balanceBuckets(smallest, largest, controllerDevices, deviceCount));
}
- CompletableFuture<Void> balanceRolesFuture = CompletableFuture.allOf(
- balanceBucketsFutures.toArray(new CompletableFuture[balanceBucketsFutures.size()]));
-
- Futures.getUnchecked(balanceRolesFuture);
+ return balanceBucketsFutures;
}
+ /**
+ * Finds node with the minimum/maximum devices from a list of nodes.
+ *
+ * @param min true: minimum, false: maximum
+ * @param controllerDevices controller nodes to devices map
+ * @return controller node with minimum/maximum devices
+ */
private ControllerNode findBucket(boolean min,
Map<ControllerNode, Set<DeviceId>> controllerDevices) {
int xSize = min ? Integer.MAX_VALUE : -1;
@@ -241,6 +276,15 @@
return xNode;
}
+ /**
+ * Balance the node buckets by moving devices from largest to smallest node.
+ *
+ * @param smallest node that is master of the smallest number of devices
+ * @param largest node that is master of the largest number of devices
+ * @param controllerDevices controller nodes to devices map
+ * @param deviceCount number of devices mastered by controller nodes
+ * @return list of setRole futures for "moved" devices
+ */
private CompletableFuture<Void> balanceBuckets(ControllerNode smallest, ControllerNode largest,
Map<ControllerNode, Set<DeviceId>> controllerDevices,
int deviceCount) {
@@ -272,6 +316,149 @@
return CompletableFuture.allOf(setRoleFutures.toArray(new CompletableFuture[setRoleFutures.size()]));
}
+ /**
+ * Balances the nodes considering Region information.
+ *
+ * @param allControllerDevices controller nodes to devices map
+ * @return true: nodes balanced; false: nodes not balanced
+ */
+ private boolean balanceRolesUsingRegions(Map<ControllerNode, Set<DeviceId>> allControllerDevices) {
+ Set<Region> regions = regionService.getRegions();
+ if (regions.isEmpty()) {
+ return false; // no balancing was done using regions.
+ }
+
+ // handle nodes belonging to regions
+ Set<ControllerNode> nodesInRegions = Sets.newHashSet();
+ for (Region region : regions) {
+ Map<ControllerNode, Set<DeviceId>> activeRegionControllers =
+ balanceRolesInRegion(region, allControllerDevices);
+ nodesInRegions.addAll(activeRegionControllers.keySet());
+ }
+
+ // handle nodes not belonging to any region
+ Set<ControllerNode> nodesNotInRegions = Sets.difference(allControllerDevices.keySet(), nodesInRegions);
+ if (!nodesNotInRegions.isEmpty()) {
+ int deviceCount = 0;
+ Map<ControllerNode, Set<DeviceId>> controllerDevicesNotInRegions = new HashMap<>();
+ for (ControllerNode controllerNode: nodesNotInRegions) {
+ controllerDevicesNotInRegions.put(controllerNode, allControllerDevices.get(controllerNode));
+ deviceCount += allControllerDevices.get(controllerNode).size();
+ }
+ // Now re-balance the buckets until they are roughly even.
+ List<CompletableFuture<Void>> balanceBucketsFutures =
+ balanceControllerNodes(controllerDevicesNotInRegions, deviceCount);
+
+ CompletableFuture<Void> balanceRolesFuture = CompletableFuture.allOf(
+ balanceBucketsFutures.toArray(new CompletableFuture[balanceBucketsFutures.size()]));
+
+ Futures.getUnchecked(balanceRolesFuture);
+ }
+ return true; // balancing was done using regions.
+ }
+
+ /**
+ * Balances the nodes in specified region.
+ *
+ * @param region region in which nodes are to be balanced
+ * @param allControllerDevices controller nodes to devices map
+ * @return controller nodes that were balanced
+ */
+ private Map<ControllerNode, Set<DeviceId>> balanceRolesInRegion(Region region,
+ Map<ControllerNode, Set<DeviceId>> allControllerDevices) {
+
+ // retrieve all devices associated with specified region
+ Set<DeviceId> devicesInRegion = regionService.getRegionDevices(region.id());
+ log.info("Region {} has {} devices.", region.id(), devicesInRegion.size());
+ if (devicesInRegion.isEmpty()) {
+ return new HashMap<>(); // no devices in this region, so nothing to balance.
+ }
+
+ List<Set<NodeId>> mastersList = region.masters();
+ log.info("Region {} has {} sets of masters.", region.id(), mastersList.size());
+ if (mastersList.isEmpty()) {
+ // TODO handle devices that belong to a region, which has no masters defined
+ return new HashMap<>(); // for now just leave devices alone
+ }
+
+ // get the region's preferred set of masters
+ Set<DeviceId> devicesInMasters = Sets.newHashSet();
+ Map<ControllerNode, Set<DeviceId>> regionalControllerDevices =
+ getRegionsPreferredMasters(region, devicesInMasters, allControllerDevices);
+
+ // Now re-balance the buckets until they are roughly even.
+ List<CompletableFuture<Void>> balanceBucketsFutures =
+ balanceControllerNodes(regionalControllerDevices, devicesInMasters.size());
+
+ // handle devices that are not currently mastered by the master node set
+ Set<DeviceId> devicesNotMasteredWithControllers = Sets.difference(devicesInRegion, devicesInMasters);
+ if (!devicesNotMasteredWithControllers.isEmpty()) {
+ // active controllers in master node set are already balanced, just
+ // assign device mastership in sequence
+ List<ControllerNode> sorted = new ArrayList<>(regionalControllerDevices.keySet());
+ Collections.sort(sorted, (o1, o2) ->
+ ((Integer) (regionalControllerDevices.get(o1)).size())
+ .compareTo((Integer) (regionalControllerDevices.get(o2)).size()));
+ int deviceIndex = 0;
+ for (DeviceId deviceId : devicesNotMasteredWithControllers) {
+ ControllerNode cnode = sorted.get(deviceIndex % sorted.size());
+ balanceBucketsFutures.add(setRole(cnode.id(), deviceId, MASTER));
+ regionalControllerDevices.get(cnode).add(deviceId);
+ deviceIndex++;
+ }
+ }
+
+ CompletableFuture<Void> balanceRolesFuture = CompletableFuture.allOf(
+ balanceBucketsFutures.toArray(new CompletableFuture[balanceBucketsFutures.size()]));
+
+ Futures.getUnchecked(balanceRolesFuture);
+
+ // update the map before returning
+ regionalControllerDevices.forEach((controllerNode, deviceIds) -> {
+ regionalControllerDevices.put(controllerNode, new HashSet<>(getDevicesOf(controllerNode.id())));
+ });
+
+ return regionalControllerDevices;
+ }
+
+ /**
+ * Get region's preferred set of master nodes - the first master node set that has at
+ * least one active node.
+ *
+ * @param region region for which preferred set of master nodes is requested
+ * @param devicesInMasters device set to track devices in preferred set of master nodes
+ * @param allControllerDevices controller nodes to devices map
+ * @return region's preferred master nodes (and devices that use them as masters)
+ */
+ private Map<ControllerNode, Set<DeviceId>> getRegionsPreferredMasters(Region region,
+ Set<DeviceId> devicesInMasters,
+ Map<ControllerNode, Set<DeviceId>> allControllerDevices) {
+ Map<ControllerNode, Set<DeviceId>> regionalControllerDevices = new HashMap<>();
+ int listIndex = 0;
+ for (Set<NodeId> masterSet: region.masters()) {
+ log.info("Region {} masters set {} has {} nodes.",
+ region.id(), listIndex, masterSet.size());
+ if (masterSet.isEmpty()) { // nothing on this level
+ listIndex++;
+ continue;
+ }
+ // Create buckets reflecting current ownership.
+ for (NodeId nodeId : masterSet) {
+ if (clusterService.getState(nodeId) == ACTIVE) {
+ ControllerNode controllerNode = clusterService.getNode(nodeId);
+ Set<DeviceId> devicesOf = new HashSet<>(allControllerDevices.get(controllerNode));
+ regionalControllerDevices.put(controllerNode, devicesOf);
+ devicesInMasters.addAll(devicesOf);
+ log.info("Active Node {} has {} devices.", nodeId, devicesOf.size());
+ }
+ }
+ if (!regionalControllerDevices.isEmpty()) {
+ break; // now have a set of >0 active controllers
+ }
+ listIndex++; // keep on looking
+ }
+ return regionalControllerDevices;
+ }
public class InternalDelegate implements MastershipStoreDelegate {
@Override