Renamed the Mastership API to use Registry terminology
diff --git a/src/main/java/net/onrc/onos/registry/controller/IControllerRegistryService.java b/src/main/java/net/onrc/onos/registry/controller/IControllerRegistryService.java
new file mode 100644
index 0000000..f2794cc
--- /dev/null
+++ b/src/main/java/net/onrc/onos/registry/controller/IControllerRegistryService.java
@@ -0,0 +1,48 @@
+package net.onrc.onos.registry.controller;
+
+import java.util.Collection;
+import java.util.Map;
+
+import net.floodlightcontroller.core.module.IFloodlightService;
+
+public interface IControllerRegistryService extends IFloodlightService {
+
+ // Callback for all mastership changes.
+ // Change callback is called when mastership is acquired or released
+ public interface ControlChangeCallback {
+ public void controlChanged(long dpid, boolean hasControl);
+ }
+
+ // Acquire mastership for a switch.
+ public void requestControl(long dpid, ControlChangeCallback cb) throws Exception;
+
+ // Release mastership for a switch
+ public void releaseControl(long dpid);
+
+ // Check if I am the master of a switch. This is a nonblocking call that checks if the caller is a
+ public boolean hasControl(long dpid);
+
+ // Set/Get mastership identifier.
+ // This is typically a unique identifier of the controller that does not change across restarts
+ public void setMastershipId (String id);
+ public String getMastershipId ();
+
+ /**
+ * Register a controller to the ONOS cluster
+ * @param controller A string identifying the controller
+ */
+ public void registerController(String controllerId) throws RegistryException;
+
+ /**
+ * Get all controllers in the cluster
+ * @return
+ */
+ public Collection<String> getAllControllers() throws RegistryException;
+
+
+ public String getControllerForSwitch(long dpid) throws RegistryException;
+
+ public Collection<Map<String, String>> getAllSwitches();
+
+ public Collection<Long> getSwitchesControlledByController(String controllerId);
+}
diff --git a/src/main/java/net/onrc/onos/registry/controller/IMastershipHelper.java b/src/main/java/net/onrc/onos/registry/controller/IMastershipHelper.java
new file mode 100644
index 0000000..9e07d46
--- /dev/null
+++ b/src/main/java/net/onrc/onos/registry/controller/IMastershipHelper.java
@@ -0,0 +1,27 @@
+package net.onrc.onos.registry.controller;
+
+public interface IMastershipHelper {
+
+ // Callback for all mastership changes.
+ // Change callback is called when mastership is acquired or released
+ public interface MastershipCallback {
+ public void changeCallback(long dpid);
+ }
+
+ // Set/get mastership identifier. This is used to set the unique identifier of the controller that is asking for mastership.
+ // It needs to be set first before any mastership call can be made
+ public void setMastershipId (String id);
+ public String getMastershipId ();
+
+ // Request mastership for a switch. Our request for mastership remains in a queue. If we win mastership, the callback
+ // is called. This call is non-blocking and can be called from the packet processing context as well.
+ public void requestMastership(long dpid, MastershipCallback cb);
+
+ // Release mastership for a switch. If we are the master, then the mastership will be released and given to the next
+ // controller who had requested mastership. If we are not the master our request for mastership will be
+ // removed from the queue.
+ public void releaseMastership(long dpid);
+
+ // Check if I am the master of a switch and return true if I am the master.
+ public boolean amMaster(long dpid);
+}
diff --git a/src/main/java/net/onrc/onos/registry/controller/MastershipHelper.java b/src/main/java/net/onrc/onos/registry/controller/MastershipHelper.java
new file mode 100644
index 0000000..2fcd002
--- /dev/null
+++ b/src/main/java/net/onrc/onos/registry/controller/MastershipHelper.java
@@ -0,0 +1,35 @@
+package net.onrc.onos.registry.controller;
+
+public class MastershipHelper implements IMastershipHelper {
+
+ @Override
+ public void setMastershipId(String id) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public String getMastershipId() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void requestMastership(long dpid, MastershipCallback cb) {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ @Override
+ public void releaseMastership(long dpid) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public boolean amMaster(long dpid) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/registry/controller/RegistryException.java b/src/main/java/net/onrc/onos/registry/controller/RegistryException.java
new file mode 100644
index 0000000..3b237c2
--- /dev/null
+++ b/src/main/java/net/onrc/onos/registry/controller/RegistryException.java
@@ -0,0 +1,28 @@
+package net.onrc.onos.registry.controller;
+
+public class RegistryException extends Exception {
+
+ private static final long serialVersionUID = -8276300722010217913L;
+
+ /*
+ public RegistryException() {
+ // TODO Auto-generated constructor stub
+ }
+
+
+ public RegistryException(String message) {
+ super(message);
+ // TODO Auto-generated constructor stub
+ }
+
+ public RegistryException(Throwable cause) {
+ super(cause);
+ // TODO Auto-generated constructor stub
+ }
+ */
+
+ public RegistryException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/registry/controller/RegistryManager.java b/src/main/java/net/onrc/onos/registry/controller/RegistryManager.java
new file mode 100644
index 0000000..84f17c3
--- /dev/null
+++ b/src/main/java/net/onrc/onos/registry/controller/RegistryManager.java
@@ -0,0 +1,359 @@
+package net.onrc.onos.registry.controller;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
+import net.floodlightcontroller.core.module.IFloodlightModule;
+import net.floodlightcontroller.core.module.IFloodlightService;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.openflow.util.HexString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.netflix.curator.RetryPolicy;
+import com.netflix.curator.framework.CuratorFramework;
+import com.netflix.curator.framework.CuratorFrameworkFactory;
+import com.netflix.curator.framework.api.CuratorWatcher;
+import com.netflix.curator.framework.recipes.cache.ChildData;
+import com.netflix.curator.framework.recipes.cache.PathChildrenCache;
+import com.netflix.curator.framework.recipes.cache.PathChildrenCache.StartMode;
+import com.netflix.curator.framework.recipes.leader.LeaderLatch;
+import com.netflix.curator.framework.recipes.leader.Participant;
+import com.netflix.curator.retry.RetryOneTime;
+
+public class RegistryManager implements IFloodlightModule, IControllerRegistryService {
+
+ protected static Logger log = LoggerFactory.getLogger(RegistryManager.class);
+ protected String mastershipId = null;
+
+ //TODO read this from configuration
+ protected String connectionString = "localhost:2181";
+ private final String namespace = "onos";
+ private final String switchLatchesPath = "/switches";
+
+ protected CuratorFramework client;
+
+ private final String controllerPath = "/controllers";
+ protected PathChildrenCache controllerCache;
+
+ protected Map<String, LeaderLatch> switchLatches;
+ protected Map<String, ControlChangeCallback> switchCallbacks;
+
+ protected boolean moduleEnabled = false;
+
+ protected class ParamaterizedCuratorWatcher implements CuratorWatcher {
+ private String dpid;
+ private boolean isLeader = false;
+ private String latchPath;
+
+ public ParamaterizedCuratorWatcher(String dpid, String latchPath){
+ this.dpid = dpid;
+ this.latchPath = latchPath;
+ }
+
+ @Override
+ public synchronized void process(WatchedEvent event) throws Exception {
+ log.debug("Watch Event: {}", event);
+
+ LeaderLatch latch = switchLatches.get(dpid);
+
+ if (event.getState() == KeeperState.Disconnected){
+ if (isLeader) {
+ log.debug("Disconnected while leader - lost leadership for {}", dpid);
+
+ isLeader = false;
+ switchCallbacks.get(dpid).controlChanged(HexString.toLong(dpid), false);
+ }
+ return;
+ }
+
+ try {
+
+ Participant leader = latch.getLeader();
+
+ if (leader.getId().equals(mastershipId) && !isLeader){
+ log.debug("Became leader for {}", dpid);
+
+ isLeader = true;
+ switchCallbacks.get(dpid).controlChanged(HexString.toLong(dpid), true);
+ }
+ else if (!leader.getId().equals(mastershipId) && isLeader){
+ log.debug("Lost leadership for {}", dpid);
+
+ isLeader = false;
+ switchCallbacks.get(dpid).controlChanged(HexString.toLong(dpid), false);
+ }
+ } catch (Exception e){
+ if (isLeader){
+ log.debug("Exception checking leadership status. Assume leadship lost for {}",
+ dpid);
+
+ isLeader = false;
+ switchCallbacks.get(dpid).controlChanged(HexString.toLong(dpid), false);
+ }
+ }
+
+ client.getChildren().usingWatcher(this).inBackground().forPath(latchPath);
+ //client.getChildren().usingWatcher(this).forPath(latchPath);
+ }
+ }
+
+
+ @Override
+ public void requestControl(long dpid, ControlChangeCallback cb) throws Exception {
+
+ if (!moduleEnabled) return;
+
+ if (mastershipId == null){
+ throw new RuntimeException("Must set mastershipId before calling aquireMastership");
+ }
+
+ String dpidStr = HexString.toHexString(dpid);
+ String latchPath = switchLatchesPath + "/" + dpidStr;
+
+ if (switchLatches.get(dpidStr) != null){
+ throw new RuntimeException("Leader election for switch " + dpidStr +
+ "is already running");
+ }
+
+ LeaderLatch latch = new LeaderLatch(client, latchPath, mastershipId);
+ switchLatches.put(dpidStr, latch);
+ switchCallbacks.put(dpidStr, cb);
+
+ try {
+ //client.getChildren().usingWatcher(watcher).inBackground().forPath(singleLatchPath);
+ client.getChildren().usingWatcher(
+ new ParamaterizedCuratorWatcher(dpidStr, latchPath))
+ .inBackground().forPath(latchPath);
+ latch.start();
+ } catch (Exception e) {
+ log.warn("Error starting leader latch: {}", e.getMessage());
+ throw e;
+ }
+
+ }
+
+ @Override
+ public void releaseControl(long dpid) {
+ if (!moduleEnabled) return;
+
+ String dpidStr = HexString.toHexString(dpid);
+
+ LeaderLatch latch = switchLatches.get(dpidStr);
+ if (latch == null) {
+ log.debug("Trying to release mastership for switch we are not contesting");
+ return;
+ }
+
+ try {
+ latch.close();
+ } catch (IOException e) {
+ //I think it's OK not to do anything here. Either the node got deleted correctly,
+ //or the connection went down and the node got deleted.
+ } finally {
+ switchLatches.remove(dpidStr);
+ switchCallbacks.remove(dpidStr);
+ }
+ }
+
+ @Override
+ public boolean hasControl(long dpid) {
+ if (!moduleEnabled) return false;
+
+ LeaderLatch latch = switchLatches.get(HexString.toHexString(dpid));
+
+ if (latch == null) {
+ log.warn("No leader latch for dpid {}", HexString.toHexString(dpid));
+ return false;
+ }
+
+ try {
+ return latch.getLeader().getId().equals(mastershipId);
+ } catch (Exception e) {
+ //TODO swallow exception?
+ return false;
+ }
+ }
+
+ @Override
+ public void setMastershipId(String id) {
+ mastershipId = id;
+ }
+
+ @Override
+ public String getMastershipId() {
+ return mastershipId;
+ }
+
+ @Override
+ public Collection<String> getAllControllers() throws RegistryException {
+ if (!moduleEnabled) return null;
+
+ log.debug("Getting all controllers");
+
+ List<String> controllers = new ArrayList<String>();
+ for (ChildData data : controllerCache.getCurrentData()){
+
+ String d = null;
+ try {
+ d = new String(data.getData(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RegistryException("Error encoding string", e);
+ }
+
+ controllers.add(d);
+ }
+ return controllers;
+ }
+
+ @Override
+ public void registerController(String id) throws RegistryException {
+ if (!moduleEnabled) return;
+
+ byte bytes[] = null;
+ try {
+ bytes = id.getBytes("UTF-8");
+ } catch (UnsupportedEncodingException e1) {
+ throw new RegistryException("Error encoding string", e1);
+ }
+
+ String path = controllerPath + "/" + id;
+
+ log.info("Registering controller with id {}", id);
+
+ //Create ephemeral node in controller registry
+ try {
+ client.create().withProtection().withMode(CreateMode.EPHEMERAL)
+ .forPath(path, bytes);
+ } catch (Exception e) {
+ throw new RegistryException("Error contacting the Zookeeper service", e);
+ }
+ }
+
+ @Override
+ public String getControllerForSwitch(long dpid) throws RegistryException {
+ if (!moduleEnabled) return null;
+ // TODO Work out how we should store this controller/switch data.
+ // The leader latch might be a index to the /controllers collections
+ // which holds more info on the controller (how to talk to it for example).
+
+
+ String strDpid = HexString.toHexString(dpid);
+ LeaderLatch latch = switchLatches.get(strDpid);
+
+ if (latch == null){
+ log.warn("Tried to get controller for non-existent switch");
+ return null;
+ }
+
+ Participant leader = null;
+ try {
+ leader = latch.getLeader();
+ } catch (Exception e) {
+ throw new RegistryException("Error contacting the Zookeeper service", e);
+ }
+
+ return leader.getId();
+ }
+
+ @Override
+ public Collection<Long> getSwitchesControlledByController(String controllerId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+
+ @Override
+ public Collection<Map<String, String>> getAllSwitches() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /*
+ * IFloodlightModule
+ */
+
+ @Override
+ public Collection<Class<? extends IFloodlightService>> getModuleServices() {
+ Collection<Class<? extends IFloodlightService>> l =
+ new ArrayList<Class<? extends IFloodlightService>>();
+ l.add(IControllerRegistryService.class);
+ return l;
+ }
+
+ @Override
+ public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() {
+ Map<Class<? extends IFloodlightService>, IFloodlightService> m =
+ new HashMap<Class<? extends IFloodlightService>, IFloodlightService>();
+ m.put(IControllerRegistryService.class, this);
+ return m;
+ }
+
+ @Override
+ public Collection<Class<? extends IFloodlightService>> getModuleDependencies() {
+ // no module dependencies
+ return null;
+ }
+
+ @Override
+ public void init (FloodlightModuleContext context) throws FloodlightModuleException {
+
+ //Read config to see if we should try and connect to zookeeper
+ Map<String, String> configOptions = context.getConfigParams(this);
+ String enableZookeeper = configOptions.get("enableZookeeper");
+ if (enableZookeeper != null) {
+ log.info("Enabling Mastership module - requires Zookeeper connection");
+ moduleEnabled = true;
+ }
+ else {
+ log.info("Mastership module is disabled");
+ return;
+ }
+
+ try {
+ String localHostname = java.net.InetAddress.getLocalHost().getHostName();
+ mastershipId = localHostname;
+ log.debug("Setting mastership id to {}", mastershipId);
+ } catch (UnknownHostException e) {
+ // TODO Handle this exception
+ e.printStackTrace();
+ }
+
+ switchLatches = new HashMap<String, LeaderLatch>();
+ switchCallbacks = new HashMap<String, ControlChangeCallback>();
+
+ //RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ RetryPolicy retryPolicy = new RetryOneTime(0);
+ client = CuratorFrameworkFactory.newClient(connectionString, retryPolicy);
+
+ client.start();
+
+ client = client.usingNamespace(namespace);
+
+ controllerCache = new PathChildrenCache(client, controllerPath, true);
+
+ try {
+ controllerCache.start(StartMode.BUILD_INITIAL_CACHE);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void startUp (FloodlightModuleContext context) {
+ // Nothing to be done on startup
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/registry/controller/RegistryRouteResource.java b/src/main/java/net/onrc/onos/registry/controller/RegistryRouteResource.java
new file mode 100644
index 0000000..5dadafa
--- /dev/null
+++ b/src/main/java/net/onrc/onos/registry/controller/RegistryRouteResource.java
@@ -0,0 +1,32 @@
+package net.onrc.onos.registry.controller;
+
+import org.restlet.resource.ServerResource;
+import org.restlet.resource.Get;
+import org.restlet.resource.Post;
+import org.restlet.resource.Delete;
+
+import org.slf4j.LoggerFactory;
+import org.slf4j.Logger;
+
+public class RegistryRouteResource extends ServerResource {
+
+ protected static Logger log = LoggerFactory.getLogger(RegistryRouteResource.class);
+
+ @Get
+ public String get(String fmJson) {
+ // TODO
+ return null;
+ }
+
+ @Post
+ public String store (String fmJson) {
+ //TODO
+ return null;
+ }
+
+ @Delete
+ public String delete (String fmJson) {
+ //TODO
+ return null;
+ }
+}
diff --git a/src/main/java/net/onrc/onos/registry/controller/RegistryRunner.java b/src/main/java/net/onrc/onos/registry/controller/RegistryRunner.java
new file mode 100644
index 0000000..8522893
--- /dev/null
+++ b/src/main/java/net/onrc/onos/registry/controller/RegistryRunner.java
@@ -0,0 +1,72 @@
+package net.onrc.onos.registry.controller;
+
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
+import net.onrc.onos.registry.controller.IControllerRegistryService.ControlChangeCallback;
+
+import org.openflow.util.HexString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used for lightweight testing of the mastership module without having
+ * to load up the entire ONOS.
+ * @author jono
+ *
+ */
+public class RegistryRunner {
+ protected static Logger log = LoggerFactory.getLogger(RegistryRunner.class);
+
+ public static void main(String args[]){
+ FloodlightModuleContext fmc = new FloodlightModuleContext();
+ RegistryManager rm = new RegistryManager();
+
+ fmc.addConfigParam(rm, "enableZookeeper", "true");
+
+ String id = null;
+ if (args.length > 0){
+ id = args[0];
+ log.info("Using unique id: {}", id);
+ }
+
+ try {
+ rm.init(fmc);
+ rm.startUp(fmc);
+
+ if (id != null){
+ rm.setMastershipId(id);
+ }
+
+ rm.requestControl(1L,
+ new ControlChangeCallback(){
+ @Override
+ public void controlChanged(long dpid, boolean isMaster) {
+ if (isMaster){
+ log.debug("Callback for becoming master for {}", HexString.toHexString(dpid));
+ }
+ else {
+ log.debug("Callback for losing mastership for {}", HexString.toHexString(dpid));
+ }
+ }
+ });
+
+ rm.registerController(id);
+
+ Thread.sleep(5000);
+
+ //"Server" loop
+ while (true) {
+ Thread.sleep(60000);
+ }
+
+ } catch (FloodlightModuleException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ log.debug("is master: {}", rm.hasControl(1L));
+ }
+}
diff --git a/src/main/java/net/onrc/onos/registry/controller/RegistryWebRoutable.java b/src/main/java/net/onrc/onos/registry/controller/RegistryWebRoutable.java
new file mode 100644
index 0000000..8b5123b
--- /dev/null
+++ b/src/main/java/net/onrc/onos/registry/controller/RegistryWebRoutable.java
@@ -0,0 +1,23 @@
+package net.onrc.onos.registry.controller;
+
+import org.restlet.Context;
+import org.restlet.Restlet;
+import org.restlet.routing.Router;
+
+import net.floodlightcontroller.restserver.RestletRoutable;
+
+public class RegistryWebRoutable implements RestletRoutable {
+
+ @Override
+ public Restlet getRestlet(Context context) {
+ Router router = new Router(context);
+ router.attach("/json", RegistryRouteResource.class);
+ return router;
+ }
+
+ @Override
+ public String basePath() {
+ return "/wm/registry";
+ }
+
+}