Fix balance-masters functionality in the new LeadershipService based device mastership store
Change-Id: I9f64d514cee7d5a5383fd4c2fa30a8616c97785c
diff --git a/core/store/dist/src/main/java/org/onosproject/store/cluster/impl/HazelcastLeadershipService.java b/core/store/dist/src/main/java/org/onosproject/store/cluster/impl/HazelcastLeadershipService.java
index ddd0b9e..d5dc43e 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/cluster/impl/HazelcastLeadershipService.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/cluster/impl/HazelcastLeadershipService.java
@@ -584,4 +584,14 @@
public List<NodeId> getCandidates(String path) {
return null;
}
+
+ @Override
+ public boolean stepdown(String path) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean makeTopCandidate(String path, NodeId nodeId) {
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/core/store/dist/src/main/java/org/onosproject/store/consistent/impl/DistributedLeadershipManager.java b/core/store/dist/src/main/java/org/onosproject/store/consistent/impl/DistributedLeadershipManager.java
index 0b2b4cb..74dae01 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/consistent/impl/DistributedLeadershipManager.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/consistent/impl/DistributedLeadershipManager.java
@@ -34,6 +34,7 @@
import org.onosproject.store.service.Versioned;
import org.slf4j.Logger;
+import java.util.ArrayList;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
@@ -48,7 +49,6 @@
import static com.google.common.base.Preconditions.checkArgument;
import static org.onlab.util.Tools.groupedThreads;
import static org.slf4j.LoggerFactory.getLogger;
-
import static org.onosproject.cluster.ControllerNode.State.ACTIVE;
import static org.onosproject.cluster.ControllerNode.State.INACTIVE;
@@ -210,7 +210,7 @@
candidateList.add(localNodeId);
if (candidateMap.replace(path, candidates.version(), candidateList)) {
Versioned<List<NodeId>> newCandidates = candidateMap.get(path);
- notifyCandidateAdded(
+ notifyCandidateUpdated(
path, candidateList, newCandidates.version(), newCandidates.creationTime());
} else {
rerunForLeadership(path);
@@ -221,7 +221,7 @@
List<NodeId> candidateList = ImmutableList.of(localNodeId);
if ((candidateMap.putIfAbsent(path, candidateList) == null)) {
Versioned<List<NodeId>> newCandidates = candidateMap.get(path);
- notifyCandidateAdded(path, candidateList, newCandidates.version(), newCandidates.creationTime());
+ notifyCandidateUpdated(path, candidateList, newCandidates.version(), newCandidates.creationTime());
} else {
rerunForLeadership(path);
return;
@@ -270,6 +270,27 @@
}
@Override
+ public boolean stepdown(String path) {
+ if (!activeTopics.contains(path)) {
+ return false;
+ }
+
+ try {
+ Versioned<NodeId> leader = leaderMap.get(path);
+ if (leader != null && Objects.equals(leader.value(), localNodeId)) {
+ if (leaderMap.remove(path, leader.version())) {
+ log.info("Gave up leadership for {}", path);
+ notifyRemovedLeader(path, localNodeId, leader.version(), leader.creationTime());
+ return true;
+ }
+ }
+ } catch (Exception e) {
+ log.warn("Error executing stepdown for {}", path, e);
+ }
+ return false;
+ }
+
+ @Override
public void addListener(LeadershipEventListener listener) {
listenerRegistry.addListener(listener);
}
@@ -279,6 +300,28 @@
listenerRegistry.removeListener(listener);
}
+ @Override
+ public boolean makeTopCandidate(String path, NodeId nodeId) {
+ Versioned<List<NodeId>> candidates = candidateMap.get(path);
+ if (candidates == null || !candidates.value().contains(nodeId)) {
+ return false;
+ }
+ if (nodeId.equals(candidates.value().get(0))) {
+ return true;
+ }
+ List<NodeId> currentRoster = candidates.value();
+ List<NodeId> newRoster = new ArrayList<>(currentRoster.size());
+ newRoster.add(nodeId);
+ currentRoster.stream().filter(id -> !nodeId.equals(id)).forEach(newRoster::add);
+ boolean updated = candidateMap.replace(path, candidates.version(), newRoster);
+ if (updated) {
+ Versioned<List<NodeId>> newCandidates = candidateMap.get(path);
+ notifyCandidateUpdated(
+ path, newCandidates.value(), newCandidates.version(), newCandidates.creationTime());
+ }
+ return updated;
+ }
+
private void tryLeaderLock(String path) {
if (!activeTopics.contains(path)) {
return;
@@ -334,7 +377,7 @@
}
}
- private void notifyCandidateAdded(
+ private void notifyCandidateUpdated(
String path, List<NodeId> candidates, long epoch, long electedTime) {
Leadership newInfo = new Leadership(path, candidates, epoch, electedTime);
final MutableBoolean updated = new MutableBoolean(false);
diff --git a/core/store/dist/src/main/java/org/onosproject/store/mastership/impl/ConsistentDeviceMastershipStore.java b/core/store/dist/src/main/java/org/onosproject/store/mastership/impl/ConsistentDeviceMastershipStore.java
index 5606c60..d7d2bc0 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/mastership/impl/ConsistentDeviceMastershipStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/mastership/impl/ConsistentDeviceMastershipStore.java
@@ -93,6 +93,8 @@
new MessageSubject("mastership-store-device-role-query");
private static final MessageSubject ROLE_RELINQUISH_SUBJECT =
new MessageSubject("mastership-store-device-role-relinquish");
+ private static final MessageSubject MASTERSHIP_RELINQUISH_SUBJECT =
+ new MessageSubject("mastership-store-device-mastership-relinquish");
private static final Pattern DEVICE_MASTERSHIP_TOPIC_PATTERN =
Pattern.compile("device:(.*)");
@@ -111,6 +113,7 @@
.register(KryoNamespaces.API)
.register(MastershipRole.class)
.register(MastershipEvent.class)
+ .register(MastershipEvent.Type.class)
.build();
}
};
@@ -125,6 +128,11 @@
clusterCommunicator.addSubscriber(ROLE_RELINQUISH_SUBJECT,
new RoleRelinquishHandler(),
messageHandlingExecutor);
+ clusterCommunicator.addSubscriber(MASTERSHIP_RELINQUISH_SUBJECT,
+ SERIALIZER::decode,
+ this::relinquishMastership,
+ SERIALIZER::encode,
+ messageHandlingExecutor);
localNodeId = clusterService.getLocalNode().id();
leadershipService.addListener(leadershipEventListener);
@@ -135,6 +143,7 @@
public void deactivate() {
clusterCommunicator.removeSubscriber(ROLE_QUERY_SUBJECT);
clusterCommunicator.removeSubscriber(ROLE_RELINQUISH_SUBJECT);
+ clusterCommunicator.removeSubscriber(MASTERSHIP_RELINQUISH_SUBJECT);
messageHandlingExecutor.shutdown();
leadershipService.removeListener(leadershipEventListener);
@@ -237,7 +246,22 @@
checkArgument(nodeId != null, NODE_ID_NULL);
checkArgument(deviceId != null, DEVICE_ID_NULL);
- throw new UnsupportedOperationException("This operation is not supported in " + this.getClass().getName());
+ NodeId currentMaster = getMaster(deviceId);
+ if (nodeId.equals(currentMaster)) {
+ return null;
+ } else {
+ String leadershipTopic = createDeviceMastershipTopic(deviceId);
+ List<NodeId> candidates = leadershipService.getCandidates(leadershipTopic);
+ if (candidates.isEmpty()) {
+ return null;
+ }
+ if (leadershipService.makeTopCandidate(leadershipTopic, nodeId)) {
+ return relinquishMastership(deviceId);
+ } else {
+ log.warn("Failed to promote {} to mastership for {}", nodeId, deviceId);
+ }
+ }
+ return null;
}
@Override
@@ -254,7 +278,13 @@
checkArgument(nodeId != null, NODE_ID_NULL);
checkArgument(deviceId != null, DEVICE_ID_NULL);
- throw new UnsupportedOperationException("This operation is not supported in " + this.getClass().getName());
+ NodeId currentMaster = getMaster(deviceId);
+ if (!nodeId.equals(currentMaster)) {
+ return null;
+ }
+ // FIXME: This can becomes the master again unless it
+ // is demoted to the end of candidates list.
+ return relinquishMastership(deviceId);
}
@Override
@@ -294,6 +324,37 @@
return new MastershipEvent(eventType, deviceId, getNodes(deviceId));
}
+ private MastershipEvent relinquishMastership(DeviceId deviceId) {
+ checkArgument(deviceId != null, DEVICE_ID_NULL);
+
+ NodeId currentMaster = getMaster(deviceId);
+ if (currentMaster == null) {
+ return null;
+ }
+
+ if (!currentMaster.equals(localNodeId)) {
+ log.info("Forwarding request to relinquish "
+ + "mastership for device {} to {}", deviceId, currentMaster);
+ return futureGetOrElse(clusterCommunicator.sendAndReceive(
+ deviceId,
+ MASTERSHIP_RELINQUISH_SUBJECT,
+ SERIALIZER::encode,
+ SERIALIZER::decode,
+ currentMaster), null);
+ }
+
+ String leadershipTopic = createDeviceMastershipTopic(deviceId);
+ Leadership currentLeadership = leadershipService.getLeadership(leadershipTopic);
+
+ MastershipEvent.Type eventType = null;
+ if (currentLeadership != null && currentLeadership.leader().equals(localNodeId)) {
+ eventType = MastershipEvent.Type.MASTER_CHANGED;
+ }
+
+ return leadershipService.stepdown(leadershipTopic)
+ ? new MastershipEvent(eventType, deviceId, getNodes(deviceId)) : null;
+ }
+
private class RoleQueryHandler implements ClusterMessageHandler {
@Override
public void handle(ClusterMessage message) {