Merge branch 'master' of ssh://gerrit.onlab.us:29418/onos-next
Conflicts:
core/store/serializers/src/main/java/org/onlab/onos/store/serializers/KryoPoolUtil.java
Change-Id: I6a8fc884f0c0b6578762d7b439177378e838735f
diff --git a/cli/src/main/java/org/onlab/onos/cli/RolesCommand.java b/cli/src/main/java/org/onlab/onos/cli/RolesCommand.java
new file mode 100644
index 0000000..0456e4a
--- /dev/null
+++ b/cli/src/main/java/org/onlab/onos/cli/RolesCommand.java
@@ -0,0 +1,74 @@
+package org.onlab.onos.cli;
+
+import static com.google.common.collect.Lists.newArrayList;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.mastership.MastershipService;
+import org.onlab.onos.net.Device;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.device.DeviceService;
+
+
+/**
+ * Lists mastership roles of nodes for each device.
+ */
+@Command(scope = "onos", name = "roles",
+ description = "Lists mastership roles of nodes for each device.")
+public class RolesCommand extends AbstractShellCommand {
+
+ private static final String FMT_HDR = "%s: master=%s\nstandbys: %s nodes";
+ private static final String FMT_SB = "\t%s";
+
+ @Override
+ protected void execute() {
+ DeviceService deviceService = get(DeviceService.class);
+ MastershipService roleService = get(MastershipService.class);
+
+ for (Device d : getSortedDevices(deviceService)) {
+ DeviceId did = d.id();
+ printRoles(roleService, did);
+ }
+ }
+
+ /**
+ * Returns the list of devices sorted using the device ID URIs.
+ *
+ * @param service device service
+ * @return sorted device list
+ */
+ protected static List<Device> getSortedDevices(DeviceService service) {
+ List<Device> devices = newArrayList(service.getDevices());
+ Collections.sort(devices, Comparators.ELEMENT_COMPARATOR);
+ return devices;
+ }
+
+ /**
+ * Prints the role information for a device.
+ *
+ * @param deviceId the ID of the device
+ * @param master the current master
+ */
+ protected void printRoles(MastershipService service, DeviceId deviceId) {
+ List<NodeId> nodes = service.getNodesFor(deviceId);
+ NodeId first = null;
+ NodeId master = null;
+
+ if (!nodes.isEmpty()) {
+ first = nodes.get(0);
+ }
+ if (first != null &&
+ first.equals(service.getMasterFor(deviceId))) {
+ master = nodes.get(0);
+ nodes.remove(master);
+ }
+ print(FMT_HDR, deviceId, master == null ? "NONE" : master, nodes.size());
+
+ for (NodeId nid : nodes) {
+ print(FMT_SB, nid);
+ }
+ }
+}
diff --git a/cli/src/main/resources/OSGI-INF/blueprint/shell-config.xml b/cli/src/main/resources/OSGI-INF/blueprint/shell-config.xml
index 9e6d396..3c75bf9 100644
--- a/cli/src/main/resources/OSGI-INF/blueprint/shell-config.xml
+++ b/cli/src/main/resources/OSGI-INF/blueprint/shell-config.xml
@@ -13,6 +13,10 @@
<command>
<action class="org.onlab.onos.cli.NodeRemoveCommand"/>
</command>
+
+ <command>
+ <action class="org.onlab.onos.cli.RolesCommand"/>
+ </command>
<command>
<action class="org.onlab.onos.cli.MastersListCommand"/>
<completers>
diff --git a/core/api/src/main/java/org/onlab/onos/mastership/MastershipService.java b/core/api/src/main/java/org/onlab/onos/mastership/MastershipService.java
index 029e357..224bc05 100644
--- a/core/api/src/main/java/org/onlab/onos/mastership/MastershipService.java
+++ b/core/api/src/main/java/org/onlab/onos/mastership/MastershipService.java
@@ -1,5 +1,6 @@
package org.onlab.onos.mastership;
+import java.util.List;
import java.util.Set;
import org.onlab.onos.cluster.NodeId;
@@ -50,6 +51,15 @@
NodeId getMasterFor(DeviceId deviceId);
/**
+ * Returns controllers connected to a given device, in order of
+ * preference. The first entry in the list is the current master.
+ *
+ * @param deviceId the identifier of the device
+ * @return a list of controller IDs
+ */
+ List<NodeId> getNodesFor(DeviceId deviceId);
+
+ /**
* Returns the devices for which a controller is master.
*
* @param nodeId the ID of the controller
diff --git a/core/api/src/main/java/org/onlab/onos/mastership/MastershipStore.java b/core/api/src/main/java/org/onlab/onos/mastership/MastershipStore.java
index 0117d0d..5e7b0e4 100644
--- a/core/api/src/main/java/org/onlab/onos/mastership/MastershipStore.java
+++ b/core/api/src/main/java/org/onlab/onos/mastership/MastershipStore.java
@@ -1,5 +1,6 @@
package org.onlab.onos.mastership;
+import java.util.List;
import java.util.Set;
import org.onlab.onos.cluster.NodeId;
@@ -41,6 +42,15 @@
NodeId getMaster(DeviceId deviceId);
/**
+ * Returns the controllers connected to a device, in mastership-
+ * preference order.
+ *
+ * @param deviceId the device identifier
+ * @return an ordered list of controller IDs
+ */
+ List<NodeId> getNodes(DeviceId deviceId);
+
+ /**
* Returns the devices that a controller instance is master of.
*
* @param nodeId the instance identifier
@@ -48,6 +58,7 @@
*/
Set<DeviceId> getDevices(NodeId nodeId);
+
/**
* Sets a device's role for a specified controller instance.
*
diff --git a/core/api/src/test/java/org/onlab/onos/mastership/MastershipServiceAdapter.java b/core/api/src/test/java/org/onlab/onos/mastership/MastershipServiceAdapter.java
index 97b57e5..af376e8 100644
--- a/core/api/src/test/java/org/onlab/onos/mastership/MastershipServiceAdapter.java
+++ b/core/api/src/test/java/org/onlab/onos/mastership/MastershipServiceAdapter.java
@@ -4,6 +4,7 @@
import org.onlab.onos.net.DeviceId;
import org.onlab.onos.net.MastershipRole;
+import java.util.List;
import java.util.Set;
/**
@@ -46,4 +47,9 @@
public MastershipTermService requestTermService() {
return null;
}
+
+ @Override
+ public List<NodeId> getNodesFor(DeviceId deviceId) {
+ return null;
+ }
}
diff --git a/core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java b/core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java
index 59614da..4bcaff4a 100644
--- a/core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java
+++ b/core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java
@@ -3,6 +3,7 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static org.slf4j.LoggerFactory.getLogger;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -127,6 +128,11 @@
return store.getDevices(nodeId);
}
+ @Override
+ public List<NodeId> getNodesFor(DeviceId deviceId) {
+ checkNotNull(deviceId, DEVICE_ID_NULL);
+ return store.getNodes(deviceId);
+ }
@Override
public MastershipTermService requestTermService() {
diff --git a/core/store/dist/src/main/java/org/onlab/onos/store/cluster/impl/DistributedClusterStore.java.bak b/core/store/dist/src/main/java/org/onlab/onos/store/cluster/impl/DistributedClusterStore.java.bak
index 5708e77..c781b23 100644
--- a/core/store/dist/src/main/java/org/onlab/onos/store/cluster/impl/DistributedClusterStore.java.bak
+++ b/core/store/dist/src/main/java/org/onlab/onos/store/cluster/impl/DistributedClusterStore.java.bak
@@ -53,7 +53,7 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
private ClusterCommunicationAdminService clusterCommunicationAdminService;
- private final ClusterNodesDelegate nodesDelegate = new InnerNodesDelegate();
+ private final ClusterNodesDelegate nodesDelegate = new InternalNodesDelegate();
@Activate
public void activate() throws IOException {
@@ -150,7 +150,7 @@
}
// Entity to handle back calls from the connection manager.
- private class InnerNodesDelegate implements ClusterNodesDelegate {
+ private class InternalNodesDelegate implements ClusterNodesDelegate {
@Override
public DefaultControllerNode nodeDetected(NodeId nodeId, IpPrefix ip, int tcpPort) {
DefaultControllerNode node = nodes.get(nodeId);
diff --git a/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/DistributedMastershipStore.java b/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/DistributedMastershipStore.java
index e073b63..d0eae2d 100644
--- a/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/DistributedMastershipStore.java
+++ b/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/DistributedMastershipStore.java
@@ -2,6 +2,9 @@
import static org.onlab.onos.mastership.MastershipEvent.Type.MASTER_CHANGED;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -20,11 +23,15 @@
import org.onlab.onos.net.DeviceId;
import org.onlab.onos.net.MastershipRole;
import org.onlab.onos.store.common.AbstractHazelcastStore;
+import org.onlab.onos.store.common.SMap;
+import org.onlab.onos.store.serializers.KryoPoolUtil;
+import org.onlab.onos.store.serializers.KryoSerializer;
+import org.onlab.util.KryoPool;
import com.google.common.collect.ImmutableSet;
-import com.hazelcast.core.ILock;
-import com.hazelcast.core.IMap;
-import com.hazelcast.core.MultiMap;
+import com.hazelcast.core.IAtomicLong;
+
+import static org.onlab.onos.net.MastershipRole.*;
/**
* Distributed implementation of the mastership store. The store is
@@ -36,36 +43,42 @@
extends AbstractHazelcastStore<MastershipEvent, MastershipStoreDelegate>
implements MastershipStore {
- //arbitrary lock name
- private static final String LOCK = "lock";
//initial term/TTL value
private static final Integer INIT = 0;
- //devices to masters
- protected IMap<byte[], byte[]> masters;
+ //device to node roles
+ protected SMap<DeviceId, RoleValue> roleMap;
//devices to terms
- protected IMap<byte[], Integer> terms;
+ protected SMap<DeviceId, Integer> terms;
+ //last-known cluster size, used for tie-breaking when partitioning occurs
+ protected IAtomicLong clusterSize;
- //re-election related, disjoint-set structures:
- //device-nodes multiset of available nodes
- protected MultiMap<byte[], byte[]> standbys;
- //device-nodes multiset for nodes that have given up on device
- protected MultiMap<byte[], byte[]> unusable;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ClusterService clusterService;
+ @SuppressWarnings({ "unchecked", "rawtypes" })
@Override
@Activate
public void activate() {
super.activate();
- masters = theInstance.getMap("masters");
- terms = theInstance.getMap("terms");
- standbys = theInstance.getMultiMap("backups");
- unusable = theInstance.getMultiMap("unusable");
+ this.serializer = new KryoSerializer() {
+ @Override
+ protected void setupKryoPool() {
+ serializerPool = KryoPool.newBuilder()
+ .register(KryoPoolUtil.API)
- masters.addEntryListener(new RemoteMasterShipEventHandler(), true);
+ .register(RoleValue.class, new RoleValueSerializer())
+ .build()
+ .populate(1);
+ }
+ };
+
+ roleMap = new SMap(theInstance.getMap("nodeRoles"), this.serializer);
+ terms = new SMap(theInstance.getMap("terms"), this.serializer);
+ clusterSize = theInstance.getAtomicLong("clustersize");
+ // roleMap.addEntryListener(new RemoteMasterShipEventHandler(), true);
log.info("Started");
}
@@ -77,12 +90,9 @@
@Override
public MastershipRole getRole(NodeId nodeId, DeviceId deviceId) {
- byte[] did = serialize(deviceId);
- byte[] nid = serialize(nodeId);
-
- NodeId current = deserialize(masters.get(did));
+ NodeId current = getNode(MASTER, deviceId);
if (current == null) {
- if (standbys.containsEntry(did, nid)) {
+ if (isRole(STANDBY, nodeId, deviceId)) {
//was previously standby, or set to standby from master
return MastershipRole.STANDBY;
} else {
@@ -101,55 +111,79 @@
@Override
public MastershipEvent setMaster(NodeId nodeId, DeviceId deviceId) {
- byte [] did = serialize(deviceId);
- byte [] nid = serialize(nodeId);
- ILock lock = theInstance.getLock(LOCK);
- lock.lock();
+ MastershipRole role = getRole(nodeId, deviceId);
+ roleMap.lock(deviceId);
try {
- MastershipRole role = getRole(nodeId, deviceId);
+ RoleValue rv = getRoleValue(deviceId);
switch (role) {
case MASTER:
//reinforce mastership
- evict(nid, did);
+ rv.reassign(nodeId, STANDBY, NONE);
+ roleMap.put(deviceId, rv);
return null;
case STANDBY:
- //make current master standby
- byte [] current = masters.get(did);
+ NodeId current = rv.get(MASTER);
if (current != null) {
- backup(current, did);
+ //backup and replace current master
+ rv.reassign(nodeId, NONE, STANDBY);
+ rv.replace(current, nodeId, MASTER);
+ } else {
+ //no master before so just add.
+ rv.add(MASTER, nodeId);
}
- //assign specified node as new master
- masters.put(did, nid);
- evict(nid, did);
- updateTerm(did);
+ rv.reassign(nodeId, STANDBY, NONE);
+ roleMap.put(deviceId, rv);
+ updateTerm(deviceId);
return new MastershipEvent(MASTER_CHANGED, deviceId, nodeId);
case NONE:
- masters.put(did, nid);
- evict(nid, did);
- updateTerm(did);
+ rv.add(MASTER, nodeId);
+ rv.reassign(nodeId, STANDBY, NONE);
+ roleMap.put(deviceId, rv);
+ updateTerm(deviceId);
return new MastershipEvent(MASTER_CHANGED, deviceId, nodeId);
default:
log.warn("unknown Mastership Role {}", role);
return null;
}
} finally {
- lock.unlock();
+ roleMap.unlock(deviceId);
}
}
@Override
public NodeId getMaster(DeviceId deviceId) {
- return deserialize(masters.get(serialize(deviceId)));
+ return getNode(MASTER, deviceId);
+ }
+
+
+ @Override
+ public List<NodeId> getNodes(DeviceId deviceId) {
+ List<NodeId> nodes = new LinkedList<>();
+
+ //add current master to head - if there is one.
+ roleMap.lock(deviceId);
+ try {
+ RoleValue rv = getRoleValue(deviceId);
+ NodeId master = rv.get(MASTER);
+ if (master != null) {
+ nodes.add(master);
+ }
+ //We ignore NONE nodes.
+ nodes.addAll(rv.nodesOfRole(STANDBY));
+ return Collections.unmodifiableList(nodes);
+ } finally {
+ roleMap.unlock(deviceId);
+ }
}
@Override
public Set<DeviceId> getDevices(NodeId nodeId) {
ImmutableSet.Builder<DeviceId> builder = ImmutableSet.builder();
- for (Map.Entry<byte[], byte[]> entry : masters.entrySet()) {
- if (nodeId.equals(deserialize(entry.getValue()))) {
- builder.add((DeviceId) deserialize(entry.getKey()));
+ for (Map.Entry<DeviceId, RoleValue> el : roleMap.entrySet()) {
+ if (nodeId.equals(el.getValue().get(MASTER))) {
+ builder.add(el.getKey());
}
}
@@ -159,26 +193,27 @@
@Override
public MastershipRole requestRole(DeviceId deviceId) {
NodeId local = clusterService.getLocalNode().id();
- byte [] did = serialize(deviceId);
- byte [] lnid = serialize(local);
- ILock lock = theInstance.getLock(LOCK);
- lock.lock();
+ roleMap.lock(deviceId);
try {
+ RoleValue rv = getRoleValue(deviceId);
MastershipRole role = getRole(local, deviceId);
switch (role) {
case MASTER:
- evict(lnid, did);
+ rv.reassign(local, STANDBY, NONE);
+ roleMap.put(deviceId, rv);
break;
case STANDBY:
- backup(lnid, did);
- terms.putIfAbsent(did, INIT);
+ rv.reassign(local, NONE, STANDBY);
+ roleMap.put(deviceId, rv);
+ terms.putIfAbsent(deviceId, INIT);
break;
case NONE:
//claim mastership
- masters.put(did, lnid);
- evict(lnid, did);
- updateTerm(did);
+ rv.add(MASTER, local);
+ rv.reassign(local, STANDBY, NONE);
+ roleMap.put(deviceId, rv);
+ updateTerm(deviceId);
role = MastershipRole.MASTER;
break;
default:
@@ -186,128 +221,131 @@
}
return role;
} finally {
- lock.unlock();
+ roleMap.unlock(deviceId);
}
}
@Override
public MastershipTerm getTermFor(DeviceId deviceId) {
- byte[] did = serialize(deviceId);
- if ((masters.get(did) == null) ||
- (terms.get(did) == null)) {
+ RoleValue rv = getRoleValue(deviceId);
+ if ((rv.get(MASTER) == null) || (terms.get(deviceId) == null)) {
return null;
}
- return MastershipTerm.of(
- (NodeId) deserialize(masters.get(did)), terms.get(did));
+ return MastershipTerm.of(rv.get(MASTER), terms.get(deviceId));
}
@Override
public MastershipEvent setStandby(NodeId nodeId, DeviceId deviceId) {
- byte [] did = serialize(deviceId);
- byte [] nid = serialize(nodeId);
MastershipEvent event = null;
- ILock lock = theInstance.getLock(LOCK);
- lock.lock();
+ roleMap.lock(deviceId);
try {
+ RoleValue rv = getRoleValue(deviceId);
MastershipRole role = getRole(nodeId, deviceId);
switch (role) {
case MASTER:
- event = reelect(nodeId, deviceId);
- backup(nid, did);
- break;
+ event = reelect(nodeId, deviceId, rv);
+ //fall through to reinforce role
case STANDBY:
//fall through to reinforce role
case NONE:
- backup(nid, did);
+ rv.reassign(nodeId, NONE, STANDBY);
+ roleMap.put(deviceId, rv);
break;
default:
log.warn("unknown Mastership Role {}", role);
}
return event;
} finally {
- lock.unlock();
+ roleMap.unlock(deviceId);
}
}
@Override
public MastershipEvent relinquishRole(NodeId nodeId, DeviceId deviceId) {
- byte [] did = serialize(deviceId);
- byte [] nid = serialize(nodeId);
MastershipEvent event = null;
- ILock lock = theInstance.getLock(LOCK);
- lock.lock();
+ roleMap.lock(deviceId);
try {
+ RoleValue rv = getRoleValue(deviceId);
MastershipRole role = getRole(nodeId, deviceId);
switch (role) {
case MASTER:
- event = reelect(nodeId, deviceId);
- evict(nid, did);
- break;
+ event = reelect(nodeId, deviceId, rv);
+ //fall through to reinforce relinquishment
case STANDBY:
//fall through to reinforce relinquishment
case NONE:
- evict(nid, did);
+ rv.reassign(nodeId, STANDBY, NONE);
+ roleMap.put(deviceId, rv);
break;
default:
log.warn("unknown Mastership Role {}", role);
}
return event;
} finally {
- lock.unlock();
+ roleMap.unlock(deviceId);
}
}
//helper to fetch a new master candidate for a given device.
- private MastershipEvent reelect(NodeId current, DeviceId deviceId) {
- byte [] did = serialize(deviceId);
- byte [] nid = serialize(current);
+ private MastershipEvent reelect(NodeId current, DeviceId deviceId, RoleValue rv) {
//if this is an queue it'd be neater.
- byte [] backup = null;
- for (byte [] n : standbys.get(serialize(deviceId))) {
- if (!current.equals(deserialize(n))) {
+ NodeId backup = null;
+ for (NodeId n : rv.nodesOfRole(STANDBY)) {
+ if (!current.equals(n)) {
backup = n;
break;
}
}
if (backup == null) {
- masters.remove(did, nid);
+ rv.remove(MASTER, current);
+ roleMap.put(deviceId, rv);
return null;
} else {
- masters.put(did, backup);
- evict(backup, did);
- Integer term = terms.get(did);
- terms.put(did, ++term);
+ rv.replace(current, backup, MASTER);
+ rv.reassign(backup, STANDBY, NONE);
+ roleMap.put(deviceId, rv);
+ Integer term = terms.get(deviceId);
+ terms.put(deviceId, ++term);
return new MastershipEvent(
- MASTER_CHANGED, deviceId, (NodeId) deserialize(backup));
+ MASTER_CHANGED, deviceId, backup);
}
}
- //adds node to pool(s) of backups and moves them from unusable.
- private void backup(byte [] nodeId, byte [] deviceId) {
- if (!standbys.containsEntry(deviceId, nodeId)) {
- standbys.put(deviceId, nodeId);
+ //return the RoleValue structure for a device, or create one
+ private RoleValue getRoleValue(DeviceId deviceId) {
+ RoleValue value = roleMap.get(deviceId);
+ if (value == null) {
+ value = new RoleValue();
+ roleMap.put(deviceId, value);
}
- if (unusable.containsEntry(deviceId, nodeId)) {
- unusable.remove(deviceId, nodeId);
- }
+ return value;
}
- //adds node to unusable and evicts it from backup pool.
- private void evict(byte [] nodeId, byte [] deviceId) {
- if (!unusable.containsEntry(deviceId, nodeId)) {
- unusable.put(deviceId, nodeId);
+ //get first applicable node out of store-unique structure.
+ private NodeId getNode(MastershipRole role, DeviceId deviceId) {
+ RoleValue value = roleMap.get(deviceId);
+ if (value != null) {
+ return value.get(role);
}
- if (standbys.containsEntry(deviceId, nodeId)) {
- standbys.remove(deviceId, nodeId);
+ return null;
+ }
+
+ //check if node is a certain role given a device
+ private boolean isRole(
+ MastershipRole role, NodeId nodeId, DeviceId deviceId) {
+ RoleValue value = roleMap.get(deviceId);
+ if (value != null) {
+ return value.contains(role, nodeId);
}
+ return false;
}
//adds or updates term information.
- private void updateTerm(byte [] deviceId) {
+ private void updateTerm(DeviceId deviceId) {
Integer term = terms.get(deviceId);
if (term == null) {
terms.put(deviceId, INIT);
diff --git a/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/RoleValue.java b/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/RoleValue.java
new file mode 100644
index 0000000..f2c3559
--- /dev/null
+++ b/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/RoleValue.java
@@ -0,0 +1,112 @@
+package org.onlab.onos.store.mastership.impl;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.MastershipRole;
+
+/**
+ * A structure that holds node mastership roles associated with a
+ * {@link DeviceId}. This structure needs to be locked through IMap.
+ */
+public class RoleValue {
+
+ protected Map<MastershipRole, List<NodeId>> value = new HashMap<>();
+
+ public RoleValue() {
+ value.put(MastershipRole.MASTER, new LinkedList<NodeId>());
+ value.put(MastershipRole.STANDBY, new LinkedList<NodeId>());
+ value.put(MastershipRole.NONE, new LinkedList<NodeId>());
+ }
+
+ public Map<MastershipRole, List<NodeId>> value() {
+ return Collections.unmodifiableMap(value);
+ }
+
+ public List<NodeId> nodesOfRole(MastershipRole type) {
+ return value.get(type);
+ }
+
+ public NodeId get(MastershipRole type) {
+ return value.get(type).isEmpty() ? null : value.get(type).get(0);
+ }
+
+ public boolean contains(MastershipRole type, NodeId nodeId) {
+ return value.get(type).contains(nodeId);
+ }
+
+ /**
+ * Associates a node to a certain role.
+ *
+ * @param type the role
+ * @param nodeId the node ID of the node to associate
+ */
+ public void add(MastershipRole type, NodeId nodeId) {
+ List<NodeId> nodes = value.get(type);
+
+ if (!nodes.contains(nodeId)) {
+ nodes.add(nodeId);
+ }
+ }
+
+ /**
+ * Removes a node from a certain role.
+ *
+ * @param type the role
+ * @param nodeId the ID of the node to remove
+ * @return
+ */
+ public boolean remove(MastershipRole type, NodeId nodeId) {
+ List<NodeId> nodes = value.get(type);
+ if (!nodes.isEmpty()) {
+ return nodes.remove(nodeId);
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Reassigns a node from one role to another. If the node was not of the
+ * old role, it will still be assigned the new role.
+ *
+ * @param nodeId the Node ID of node changing roles
+ * @param from the old role
+ * @param to the new role
+ */
+ // might want to add anyways as default behavior
+ public void reassign(NodeId nodeId, MastershipRole from, MastershipRole to) {
+ remove(from, nodeId);
+ add(to, nodeId);
+ }
+
+ /**
+ * Replaces a node in one role with another node. Even if there is no node to
+ * replace, the new node is associated to the role.
+ *
+ * @param from the old NodeId to replace
+ * @param to the new NodeId
+ * @param type the role associated with the old NodeId
+ */
+ // might want to add anyways as default behavior
+ public void replace(NodeId from, NodeId to, MastershipRole type) {
+ remove(type, from);
+ add(type, to);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder builder = new StringBuilder();
+ for (Map.Entry<MastershipRole, List<NodeId>> el : value.entrySet()) {
+ builder.append(el.getKey().toString()).append(": [");
+ for (NodeId n : el.getValue()) {
+ builder.append(n);
+ }
+ builder.append("]\n");
+ }
+ return builder.toString();
+ }
+}
diff --git a/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/RoleValueSerializer.java b/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/RoleValueSerializer.java
new file mode 100644
index 0000000..22d1b35
--- /dev/null
+++ b/core/store/hz/cluster/src/main/java/org/onlab/onos/store/mastership/impl/RoleValueSerializer.java
@@ -0,0 +1,52 @@
+package org.onlab.onos.store.mastership.impl;
+
+import java.util.List;
+import java.util.Map;
+
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.MastershipRole;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Serializer for RoleValues used by {@link DistributedMastershipStore}.
+ */
+public class RoleValueSerializer extends Serializer<RoleValue> {
+
+ //RoleValues are assumed to hold a Map of MastershipRoles (an enum)
+ //to a List of NodeIds.
+
+ @Override
+ public RoleValue read(Kryo kryo, Input input, Class<RoleValue> type) {
+ RoleValue rv = new RoleValue();
+ int size = input.readInt();
+ for (int i = 0; i < size; i++) {
+ MastershipRole role = MastershipRole.values()[input.readInt()];
+ int s = input.readInt();
+ for (int j = 0; j < s; j++) {
+ rv.add(role, new NodeId(input.readString()));
+ }
+ }
+ return rv;
+ }
+
+ @Override
+ public void write(Kryo kryo, Output output, RoleValue type) {
+ output.writeInt(type.value().size());
+
+ for (Map.Entry<MastershipRole, List<NodeId>> el :
+ type.value().entrySet()) {
+ output.writeInt(el.getKey().ordinal());
+
+ List<NodeId> nodes = el.getValue();
+ output.writeInt(nodes.size());
+ for (NodeId n : nodes) {
+ output.writeString(n.toString());
+ }
+ }
+ }
+
+}
diff --git a/core/store/hz/cluster/src/test/java/org/onlab/onos/store/mastership/impl/DistributedMastershipStoreTest.java b/core/store/hz/cluster/src/test/java/org/onlab/onos/store/mastership/impl/DistributedMastershipStoreTest.java
index 89c4357..956b0be 100644
--- a/core/store/hz/cluster/src/test/java/org/onlab/onos/store/mastership/impl/DistributedMastershipStoreTest.java
+++ b/core/store/hz/cluster/src/test/java/org/onlab/onos/store/mastership/impl/DistributedMastershipStoreTest.java
@@ -27,6 +27,7 @@
import org.onlab.onos.mastership.MastershipTerm;
import org.onlab.onos.mastership.MastershipEvent.Type;
import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.MastershipRole;
import org.onlab.onos.store.common.StoreManager;
import org.onlab.onos.store.common.StoreService;
import org.onlab.onos.store.common.TestStoreManager;
@@ -101,7 +102,7 @@
@Test
public void getMaster() {
- assertTrue("wrong store state:", dms.masters.isEmpty());
+ assertTrue("wrong store state:", dms.roleMap.isEmpty());
testStore.put(DID1, N1, true, false, false);
assertEquals("wrong master:", N1, dms.getMaster(DID1));
@@ -110,12 +111,11 @@
@Test
public void getDevices() {
- assertTrue("wrong store state:", dms.masters.isEmpty());
+ assertTrue("wrong store state:", dms.roleMap.isEmpty());
testStore.put(DID1, N1, true, false, false);
testStore.put(DID2, N1, true, false, false);
testStore.put(DID3, N2, true, false, false);
-
assertEquals("wrong devices",
Sets.newHashSet(DID1, DID2), dms.getDevices(N1));
}
@@ -161,7 +161,7 @@
assertEquals("wrong event:", Type.MASTER_CHANGED, dms.setMaster(N2, DID2).type());
assertEquals("wrong term", MastershipTerm.of(N2, 0), dms.getTermFor(DID2));
//disconnect and reconnect - sign of failing re-election or single-instance channel
- testStore.reset(true, false, false);
+ dms.roleMap.clear();
dms.setMaster(N2, DID2);
assertEquals("wrong term", MastershipTerm.of(N2, 1), dms.getTermFor(DID2));
}
@@ -191,13 +191,15 @@
assertEquals("wrong role for node:", NONE, dms.getRole(N2, DID1));
assertEquals("wrong role for node:", NONE, dms.getRole(N1, DID1));
- assertEquals("wrong number of retired nodes", 2, dms.unusable.size());
+ assertEquals("wrong number of retired nodes", 2,
+ dms.roleMap.get(DID1).nodesOfRole(NONE).size());
//bring nodes back
assertEquals("wrong role for NONE:", MASTER, dms.requestRole(DID1));
testStore.setCurrent(CN1);
assertEquals("wrong role for NONE:", STANDBY, dms.requestRole(DID1));
- assertEquals("wrong number of backup nodes", 1, dms.standbys.size());
+ assertEquals("wrong number of backup nodes", 1,
+ dms.roleMap.get(DID1).nodesOfRole(STANDBY).size());
//NONE - nothing happens
assertNull("wrong event:", dms.relinquishRole(N1, DID2));
@@ -238,55 +240,44 @@
//helper to populate master/backup structures
public void put(DeviceId dev, NodeId node,
boolean master, boolean backup, boolean term) {
- byte [] n = serialize(node);
- byte [] d = serialize(dev);
+ RoleValue rv = dms.roleMap.get(dev);
+ if (rv == null) {
+ rv = new RoleValue();
+ }
if (master) {
- dms.masters.put(d, n);
- dms.unusable.put(d, n);
- dms.standbys.remove(d, n);
+ rv.add(MASTER, node);
+ rv.reassign(node, STANDBY, NONE);
}
if (backup) {
- dms.standbys.put(d, n);
- dms.masters.remove(d, n);
- dms.unusable.remove(d, n);
+ rv.add(STANDBY, node);
+ rv.remove(MASTER, node);
+ rv.remove(NONE, node);
}
if (term) {
- dms.terms.put(d, 0);
+ dms.terms.put(dev, 0);
}
+ dms.roleMap.put(dev, rv);
}
//a dumb utility function.
public void dump() {
- System.out.println("standbys");
- for (Map.Entry<byte [], byte []> e : standbys.entrySet()) {
- System.out.println(deserialize(e.getKey()) + ":" + deserialize(e.getValue()));
- }
- System.out.println("unusable");
- for (Map.Entry<byte [], byte []> e : unusable.entrySet()) {
- System.out.println(deserialize(e.getKey()) + ":" + deserialize(e.getValue()));
- }
- }
-
- //clears structures
- public void reset(boolean store, boolean backup, boolean term) {
- if (store) {
- dms.masters.clear();
- dms.unusable.clear();
- }
- if (backup) {
- dms.standbys.clear();
- }
- if (term) {
- dms.terms.clear();
+ for (Map.Entry<DeviceId, RoleValue> el : dms.roleMap.entrySet()) {
+ System.out.println("DID: " + el.getKey());
+ for (MastershipRole role : MastershipRole.values()) {
+ System.out.println("\t" + role.toString() + ":");
+ for (NodeId n : el.getValue().nodesOfRole(role)) {
+ System.out.println("\t\t" + n);
+ }
+ }
}
}
//increment term for a device
public void increment(DeviceId dev) {
- Integer t = dms.terms.get(serialize(dev));
+ Integer t = dms.terms.get(dev);
if (t != null) {
- dms.terms.put(serialize(dev), ++t);
+ dms.terms.put(dev, ++t);
}
}
diff --git a/core/store/hz/cluster/src/test/java/org/onlab/onos/store/mastership/impl/RoleValueTest.java b/core/store/hz/cluster/src/test/java/org/onlab/onos/store/mastership/impl/RoleValueTest.java
new file mode 100644
index 0000000..93741b7
--- /dev/null
+++ b/core/store/hz/cluster/src/test/java/org/onlab/onos/store/mastership/impl/RoleValueTest.java
@@ -0,0 +1,31 @@
+package org.onlab.onos.store.mastership.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.onlab.onos.net.MastershipRole.*;
+
+import org.junit.Test;
+import org.onlab.onos.cluster.NodeId;
+
+import com.google.common.collect.Sets;
+
+public class RoleValueTest {
+
+ private static final RoleValue RV = new RoleValue();
+
+ private static final NodeId NID1 = new NodeId("node1");
+ private static final NodeId NID2 = new NodeId("node2");
+ private static final NodeId NID3 = new NodeId("node3");
+
+ @Test
+ public void add() {
+ assertEquals("faulty initialization: ", 3, RV.value.size());
+ RV.add(MASTER, NID1);
+ RV.add(STANDBY, NID2);
+ RV.add(STANDBY, NID3);
+
+ assertEquals("wrong nodeID: ", NID1, RV.get(MASTER));
+ assertTrue("wrong nodeIDs: ",
+ Sets.newHashSet(NID3, NID2).containsAll(RV.nodesOfRole(STANDBY)));
+ }
+}
diff --git a/core/store/serializers/src/main/java/org/onlab/onos/store/serializers/KryoPoolUtil.java b/core/store/serializers/src/main/java/org/onlab/onos/store/serializers/KryoPoolUtil.java
index b229063..38c4dfd 100644
--- a/core/store/serializers/src/main/java/org/onlab/onos/store/serializers/KryoPoolUtil.java
+++ b/core/store/serializers/src/main/java/org/onlab/onos/store/serializers/KryoPoolUtil.java
@@ -21,7 +21,6 @@
import org.onlab.onos.net.HostLocation;
import org.onlab.onos.net.Link;
import org.onlab.onos.net.LinkKey;
-import org.onlab.onos.net.MastershipRole;
import org.onlab.onos.net.Port;
import org.onlab.onos.net.PortNumber;
import org.onlab.onos.net.device.DefaultDeviceDescription;
@@ -76,7 +75,6 @@
DefaultDevice.class,
DefaultDeviceDescription.class,
DefaultLinkDescription.class,
- MastershipRole.class,
Port.class,
DefaultPortDescription.class,
Element.class,
diff --git a/core/store/serializers/src/main/java/org/onlab/onos/store/serializers/MastershipRoleSerializer.java b/core/store/serializers/src/main/java/org/onlab/onos/store/serializers/MastershipRoleSerializer.java
deleted file mode 100644
index dab5aa8..0000000
--- a/core/store/serializers/src/main/java/org/onlab/onos/store/serializers/MastershipRoleSerializer.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.onlab.onos.store.serializers;
-
-import org.onlab.onos.net.MastershipRole;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.Serializer;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-
-/**
- * Kryo Serializer for {@link org.onlab.onos.net.MastershipRole}.
- */
-public class MastershipRoleSerializer extends Serializer<MastershipRole> {
-
- /**
- * Creates {@link MastershipRole} serializer instance.
- */
- public MastershipRoleSerializer() {
- // non-null, immutable
- super(false, true);
- }
-
- @Override
- public MastershipRole read(Kryo kryo, Input input, Class<MastershipRole> type) {
- final String role = kryo.readObject(input, String.class);
- return MastershipRole.valueOf(role);
- }
-
- @Override
- public void write(Kryo kryo, Output output, MastershipRole object) {
- kryo.writeObject(output, object.toString());
- }
-
-}
diff --git a/core/store/serializers/src/test/java/org/onlab/onos/store/serializers/KryoSerializerTest.java b/core/store/serializers/src/test/java/org/onlab/onos/store/serializers/KryoSerializerTest.java
index 58956d5..df58ea5 100644
--- a/core/store/serializers/src/test/java/org/onlab/onos/store/serializers/KryoSerializerTest.java
+++ b/core/store/serializers/src/test/java/org/onlab/onos/store/serializers/KryoSerializerTest.java
@@ -22,7 +22,6 @@
import org.onlab.onos.net.DeviceId;
import org.onlab.onos.net.Link;
import org.onlab.onos.net.LinkKey;
-import org.onlab.onos.net.MastershipRole;
import org.onlab.onos.net.PortNumber;
import org.onlab.onos.net.SparseAnnotations;
import org.onlab.onos.net.provider.ProviderId;
@@ -115,9 +114,6 @@
testSerialized(PIDA);
testSerialized(new NodeId("bar"));
testSerialized(MastershipTerm.of(new NodeId("foo"), 2));
- for (MastershipRole role : MastershipRole.values()) {
- testSerialized(role);
- }
}
@Test
diff --git a/core/store/trivial/src/main/java/org/onlab/onos/store/trivial/impl/SimpleMastershipStore.java b/core/store/trivial/src/main/java/org/onlab/onos/store/trivial/impl/SimpleMastershipStore.java
index aba77d0..f4b035c 100644
--- a/core/store/trivial/src/main/java/org/onlab/onos/store/trivial/impl/SimpleMastershipStore.java
+++ b/core/store/trivial/src/main/java/org/onlab/onos/store/trivial/impl/SimpleMastershipStore.java
@@ -2,9 +2,11 @@
import static org.slf4j.LoggerFactory.getLogger;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -95,6 +97,18 @@
}
@Override
+ public List<NodeId> getNodes(DeviceId deviceId) {
+ List<NodeId> nodes = new ArrayList<>();
+
+ nodes.addAll(backups);
+ if (!nodes.contains(masterMap.get(deviceId))) {
+ nodes.add(masterMap.get(deviceId));
+ }
+
+ return Collections.unmodifiableList(nodes);
+ }
+
+ @Override
public Set<DeviceId> getDevices(NodeId nodeId) {
Set<DeviceId> ids = new HashSet<>();
for (Map.Entry<DeviceId, NodeId> d : masterMap.entrySet()) {
diff --git a/tools/test/bin/onos-config b/tools/test/bin/onos-config
deleted file mode 100755
index a8ef4fe..0000000
--- a/tools/test/bin/onos-config
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-# -----------------------------------------------------------------------------
-# Remotely configures & starts ONOS for the first time.
-# -----------------------------------------------------------------------------
-
-[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
-. $ONOS_ROOT/tools/build/envDefaults
-
-remote=$ONOS_USER@${1:-$OCI}
-
-# Generate a cluster.json from the ON* environment variables
-CDEF_FILE=/tmp/cluster.json
-echo "{ \"nodes\":[" > $CDEF_FILE
-for node in $(env | sort | egrep "OC[2-9]+" | cut -d= -f2); do
- echo " { \"id\": \"$node\", \"ip\": \"$node\", \"tcpPort\": 9876 }," >> $CDEF_FILE
-done
-echo " { \"id\": \"$OC1\", \"ip\": \"$OC1\", \"tcpPort\": 9876 }" >> $CDEF_FILE
-echo "]}" >> $CDEF_FILE
-
-ssh $remote "
- sudo perl -pi.bak -e \"s/ <interface>.*</ <interface>${ONOS_NIC:-192.168.56.*}</g\" \
- $ONOS_INSTALL_DIR/$KARAF_DIST/etc/hazelcast.xml
-
- echo \"onos.ip = \$(ifconfig | grep $ONOS_NIC | cut -d: -f2 | cut -d\\ -f1)\" \
- >> $ONOS_INSTALL_DIR/$KARAF_DIST/etc/system.properties
-"
-scp -q $CDEF_FILE $remote:$ONOS_INSTALL_DIR/config/