Added ability to force mastership re-balancing between instances from the GUI.
Change-Id: I98e56deb3e2b00df630ed85b596c8e35b3d6efab
diff --git a/cli/src/main/java/org/onlab/onos/cli/BalanceMastersCommand.java b/cli/src/main/java/org/onlab/onos/cli/BalanceMastersCommand.java
index 77d0c16..9b31715 100644
--- a/cli/src/main/java/org/onlab/onos/cli/BalanceMastersCommand.java
+++ b/cli/src/main/java/org/onlab/onos/cli/BalanceMastersCommand.java
@@ -15,23 +15,8 @@
*/
package org.onlab.onos.cli;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
import org.apache.karaf.shell.commands.Command;
-import org.onlab.onos.cluster.ClusterService;
-import org.onlab.onos.cluster.ControllerNode;
import org.onlab.onos.mastership.MastershipAdminService;
-import org.onlab.onos.mastership.MastershipService;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.device.DeviceService;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import static com.google.common.collect.Lists.newArrayList;
-import static org.onlab.onos.net.MastershipRole.MASTER;
/**
* Forces device mastership rebalancing.
@@ -42,72 +27,7 @@
@Override
protected void execute() {
- ClusterService service = get(ClusterService.class);
- MastershipService mastershipService = get(MastershipService.class);
- MastershipAdminService adminService = get(MastershipAdminService.class);
-
- List<ControllerNode> nodes = newArrayList(service.getNodes());
-
- Multimap<ControllerNode, DeviceId> controllerDevices = HashMultimap.create();
-
- // Create buckets reflecting current ownership.
- for (ControllerNode node : nodes) {
- Set<DeviceId> devicesOf = mastershipService.getDevicesOf(node.id());
- controllerDevices.putAll(node, devicesOf);
- print("Node %s has %d devices.", node.id(), devicesOf.size());
- }
-
- int rounds = nodes.size();
- for (int i = 0; i < rounds; i++) {
- // Iterate over the buckets and find the smallest and the largest.
- ControllerNode smallest = findBucket(true, nodes, controllerDevices);
- ControllerNode largest = findBucket(false, nodes, controllerDevices);
- balanceBuckets(smallest, largest, controllerDevices, adminService);
- }
- }
-
- private ControllerNode findBucket(boolean min, Collection<ControllerNode> nodes,
- Multimap<ControllerNode, DeviceId> controllerDevices) {
- int xSize = min ? Integer.MAX_VALUE : -1;
- ControllerNode xNode = null;
- for (ControllerNode node : nodes) {
- int size = controllerDevices.get(node).size();
- if ((min && size < xSize) || (!min && size > xSize)) {
- xSize = size;
- xNode = node;
- }
- }
- return xNode;
- }
-
- // FIXME: enhance to better handle cases where smallest cannot take any of the devices from largest
-
- private void balanceBuckets(ControllerNode smallest, ControllerNode largest,
- Multimap<ControllerNode, DeviceId> controllerDevices,
- MastershipAdminService adminService) {
- Collection<DeviceId> minBucket = controllerDevices.get(smallest);
- Collection<DeviceId> maxBucket = controllerDevices.get(largest);
- int bucketCount = controllerDevices.keySet().size();
- int deviceCount = get(DeviceService.class).getDeviceCount();
-
- int delta = (maxBucket.size() - minBucket.size()) / 2;
- delta = Math.min(deviceCount / bucketCount, delta);
-
- if (delta > 0) {
- print("Attempting to move %d nodes from %s to %s...",
- delta, largest.id(), smallest.id());
-
- int i = 0;
- Iterator<DeviceId> it = maxBucket.iterator();
- while (it.hasNext() && i < delta) {
- DeviceId deviceId = it.next();
- print("Setting %s as the master for %s", smallest.id(), deviceId);
- adminService.setRole(smallest.id(), deviceId, MASTER);
- controllerDevices.put(smallest, deviceId);
- it.remove();
- i++;
- }
- }
+ get(MastershipAdminService.class).balanceRoles();
}
}
diff --git a/core/api/src/main/java/org/onlab/onos/mastership/MastershipAdminService.java b/core/api/src/main/java/org/onlab/onos/mastership/MastershipAdminService.java
index f2fd8f4..cb3869d 100644
--- a/core/api/src/main/java/org/onlab/onos/mastership/MastershipAdminService.java
+++ b/core/api/src/main/java/org/onlab/onos/mastership/MastershipAdminService.java
@@ -33,4 +33,10 @@
*/
void setRole(NodeId instance, DeviceId deviceId, MastershipRole role);
+ /**
+ * Balances the mastership to be shared as evenly as possibly by all
+ * online instances.
+ */
+ void balanceRoles();
+
}
diff --git a/core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java b/core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java
index 4f993c4..aaac493 100644
--- a/core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java
+++ b/core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java
@@ -15,13 +15,10 @@
*/
package org.onlab.onos.cluster.impl;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.slf4j.LoggerFactory.getLogger;
-import static org.onlab.metrics.MetricsUtil.*;
-
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.Timer.Context;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
@@ -50,8 +47,18 @@
import org.onlab.onos.net.MastershipRole;
import org.slf4j.Logger;
-import com.codahale.metrics.Timer;
-import com.codahale.metrics.Timer.Context;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Lists.newArrayList;
+import static org.onlab.metrics.MetricsUtil.startTimer;
+import static org.onlab.metrics.MetricsUtil.stopTimer;
+import static org.onlab.onos.net.MastershipRole.MASTER;
+import static org.slf4j.LoggerFactory.getLogger;
@Component(immediate = true)
@Service
@@ -198,6 +205,71 @@
return metricsService;
}
+ @Override
+ public void balanceRoles() {
+ List<ControllerNode> nodes = newArrayList(clusterService.getNodes());
+ Multimap<ControllerNode, DeviceId> controllerDevices = HashMultimap.create();
+ int deviceCount = 0;
+
+ // Create buckets reflecting current ownership.
+ for (ControllerNode node : nodes) {
+ Set<DeviceId> devicesOf = getDevicesOf(node.id());
+ deviceCount += devicesOf.size();
+ controllerDevices.putAll(node, devicesOf);
+ log.info("Node {} has {} devices.", node.id(), devicesOf.size());
+ }
+
+ int rounds = nodes.size();
+ for (int i = 0; i < rounds; i++) {
+ // Iterate over the buckets and find the smallest and the largest.
+ ControllerNode smallest = findBucket(true, nodes, controllerDevices);
+ ControllerNode largest = findBucket(false, nodes, controllerDevices);
+ balanceBuckets(smallest, largest, controllerDevices, deviceCount);
+ }
+ }
+
+ private ControllerNode findBucket(boolean min, Collection<ControllerNode> nodes,
+ Multimap<ControllerNode, DeviceId> controllerDevices) {
+ int xSize = min ? Integer.MAX_VALUE : -1;
+ ControllerNode xNode = null;
+ for (ControllerNode node : nodes) {
+ int size = controllerDevices.get(node).size();
+ if ((min && size < xSize) || (!min && size > xSize)) {
+ xSize = size;
+ xNode = node;
+ }
+ }
+ return xNode;
+ }
+
+ private void balanceBuckets(ControllerNode smallest, ControllerNode largest,
+ Multimap<ControllerNode, DeviceId> controllerDevices,
+ int deviceCount) {
+ Collection<DeviceId> minBucket = controllerDevices.get(smallest);
+ Collection<DeviceId> maxBucket = controllerDevices.get(largest);
+ int bucketCount = controllerDevices.keySet().size();
+
+ int delta = (maxBucket.size() - minBucket.size()) / 2;
+ delta = Math.min(deviceCount / bucketCount, delta);
+
+ if (delta > 0) {
+ log.info("Attempting to move {} nodes from {} to {}...", delta,
+ largest.id(), smallest.id());
+
+ int i = 0;
+ Iterator<DeviceId> it = maxBucket.iterator();
+ while (it.hasNext() && i < delta) {
+ DeviceId deviceId = it.next();
+ log.info("Setting {} as the master for {}", smallest.id(), deviceId);
+ setRole(smallest.id(), deviceId, MASTER);
+ controllerDevices.put(smallest, deviceId);
+ it.remove();
+ i++;
+ }
+ }
+ }
+
+
// Posts the specified event to the local event dispatcher.
private void post(MastershipEvent event) {
if (event != null && eventDispatcher != null) {
diff --git a/web/gui/src/main/java/org/onlab/onos/gui/TopologyViewWebSocket.java b/web/gui/src/main/java/org/onlab/onos/gui/TopologyViewWebSocket.java
index 84e2e02..7477e60 100644
--- a/web/gui/src/main/java/org/onlab/onos/gui/TopologyViewWebSocket.java
+++ b/web/gui/src/main/java/org/onlab/onos/gui/TopologyViewWebSocket.java
@@ -27,6 +27,7 @@
import org.onlab.onos.event.AbstractEventAccumulator;
import org.onlab.onos.event.Event;
import org.onlab.onos.event.EventAccumulator;
+import org.onlab.onos.mastership.MastershipAdminService;
import org.onlab.onos.mastership.MastershipEvent;
import org.onlab.onos.mastership.MastershipListener;
import org.onlab.onos.net.ConnectPoint;
@@ -233,6 +234,9 @@
requestSummary(event);
} else if (type.equals("cancelSummary")) {
cancelSummary(event);
+
+ } else if (type.equals("equalizeMasters")) {
+ equalizeMasters(event);
}
}
@@ -449,6 +453,12 @@
}
+ // Forces mastership role rebalancing.
+ private void equalizeMasters(ObjectNode event) {
+ directory.get(MastershipAdminService.class).balanceRoles();
+ }
+
+
// Adds all internal listeners.
private void addListeners() {
clusterService.addListener(clusterListener);
diff --git a/web/gui/src/main/webapp/d3Utils.js b/web/gui/src/main/webapp/d3Utils.js
index 33bba5a..0161459 100644
--- a/web/gui/src/main/webapp/d3Utils.js
+++ b/web/gui/src/main/webapp/d3Utils.js
@@ -140,12 +140,12 @@
// TODO: tune colors for light and dark themes
// Note: These colors look good on the white background. Still, need to tune for dark.
- // blue brown purple sea green brick red dark teal lime
- var lightNorm = ['#3E5780', '#78533B', '#8A2979', '#018D61', '#CB4D28', '#006D73', '#56AF00'],
- lightMute = ['#A8B8CC', '#CCB3A8', '#D19FCE', '#96D6BF', '#FFC2BD', '#8FCCCA', '#CAEAA4'],
+ // blue brown brick red sea green purple dark teal lime
+ var lightNorm = ['#3E5780', '#78533B', '#CB4D28', '#018D61', '#8A2979', '#006D73', '#56AF00'],
+ lightMute = ['#A8B8CC', '#CCB3A8', '#FFC2BD', '#96D6BF', '#D19FCE', '#8FCCCA', '#CAEAA4'],
- darkNorm = ['#3E5780', '#78533B', '#8A2979', '#018D61', '#CB4D28', '#006D73', '#56AF00'],
- darkMute = ['#A8B8CC', '#CCB3A8', '#D19FCE', '#96D6BF', '#FFC2BD', '#8FCCCA', '#CAEAA4'];
+ darkNorm = ['#3E5780', '#78533B', '#CB4D28', '#018D61', '#8A2979', '#006D73', '#56AF00'],
+ darkMute = ['#A8B8CC', '#CCB3A8', '#FFC2BD', '#96D6BF', '#D19FCE', '#8FCCCA', '#CAEAA4'];
function cat7() {
var colors = {
diff --git a/web/gui/src/main/webapp/onos.js b/web/gui/src/main/webapp/onos.js
index 53d68f6..5dfcad1 100644
--- a/web/gui/src/main/webapp/onos.js
+++ b/web/gui/src/main/webapp/onos.js
@@ -97,6 +97,7 @@
case 187: return 'equals';
case 189: return 'dash';
case 191: return 'slash';
+ case 192: return 'backQuote';
default:
if ((code >= 48 && code <= 57) ||
(code >= 65 && code <= 90)) {
diff --git a/web/gui/src/main/webapp/onosQuickHelp.js b/web/gui/src/main/webapp/onosQuickHelp.js
index 596276b..9e2a5ec 100644
--- a/web/gui/src/main/webapp/onosQuickHelp.js
+++ b/web/gui/src/main/webapp/onosQuickHelp.js
@@ -55,6 +55,7 @@
equals: '=',
dash: '-',
slash: '/',
+ backQuote: '`',
leftArrow: 'L-arrow',
upArrow: 'U-arrow',
rightArrow: 'R-arrow',
diff --git a/web/gui/src/main/webapp/topo.js b/web/gui/src/main/webapp/topo.js
index 6673b28..4018757 100644
--- a/web/gui/src/main/webapp/topo.js
+++ b/web/gui/src/main/webapp/topo.js
@@ -140,6 +140,7 @@
equals: injectStartupEvents,
dash: injectTestEvent,
+ E: [equalizeMasters, 'Equalize mastership roles'],
O: [toggleSummary, 'Toggle ONOS summary pane'],
I: [toggleInstances, 'Toggle ONOS instances pane'],
D: [toggleDetails, 'Disable / enable details pane'],
@@ -926,6 +927,11 @@
updateDeviceColors();
}
+ function equalizeMasters() {
+ flash('Equalizing master roles');
+ sendMessage('equalizeMasters');
+ }
+
function toggleSummary() {
if (!summaryPane.isVisible()) {
requestSummary();