Renamed registry and packet packages
net.onrc.onos.registry.controller.* => net.onrc.onos.core.registry.*
net.onrc.onos.packet => net.onrc.onos.core.packet
Change-Id: I595e325a2c0bab709f248dde5d84ff7b6185cf8b
diff --git a/src/main/java/net/onrc/onos/core/registry/ControllerRegistryEntry.java b/src/main/java/net/onrc/onos/core/registry/ControllerRegistryEntry.java
new file mode 100644
index 0000000..a4eee8a
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/ControllerRegistryEntry.java
@@ -0,0 +1,27 @@
+package net.onrc.onos.core.registry;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+
+public class ControllerRegistryEntry implements Comparable<ControllerRegistryEntry> {
+
+ private String controllerId;
+ private int sequenceNumber;
+
+ public ControllerRegistryEntry(String controllerId, int sequenceNumber) {
+ this.controllerId = controllerId;
+ this.sequenceNumber = sequenceNumber;
+ }
+
+ @JsonProperty("controllerId")
+ public String getControllerId(){
+ return controllerId;
+ }
+
+ @Override
+ public int compareTo(ControllerRegistryEntry o) {
+ return sequenceNumber - o.sequenceNumber;
+ //return 0;
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/ControllerService.java b/src/main/java/net/onrc/onos/core/registry/ControllerService.java
new file mode 100644
index 0000000..e1e15e9
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/ControllerService.java
@@ -0,0 +1,26 @@
+package net.onrc.onos.core.registry;
+
+
+
+//@JsonRootName("controller")
+public class ControllerService {
+
+ private String controllerId;
+
+ public ControllerService(){
+ this("");
+ }
+
+ public ControllerService(String controllerId) {
+ this.controllerId = controllerId;
+ }
+
+ public void setControllerId(String controllerId) {
+ this.controllerId = controllerId;
+ }
+
+ public String getControllerId() {
+ return controllerId;
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/IControllerRegistryService.java b/src/main/java/net/onrc/onos/core/registry/IControllerRegistryService.java
new file mode 100755
index 0000000..e271754
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/IControllerRegistryService.java
@@ -0,0 +1,159 @@
+package net.onrc.onos.core.registry;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import net.floodlightcontroller.core.module.IFloodlightService;
+
+/**
+ * A registry service that allows ONOS to register controllers and switches
+ * in a way that is global to the entire ONOS cluster. The registry is the
+ * arbiter for allowing controllers to control switches.
+ *
+ * The OVS/OF1.{2,3} fault tolerance model is a switch connects to multiple
+ * controllers, and the controllers send role requests to determine what their
+ * role is in controlling the switch.
+ *
+ * The ONOS fault tolerance model allows only a single controller to have
+ * control of a switch (MASTER role) at once. Controllers therefore need a
+ * mechanism that enables them to decide who should control a each switch.
+ * The registry service provides this mechanism.
+ *
+ * @author jono
+ */
+public interface IControllerRegistryService extends IFloodlightService {
+
+ /**
+ * Callback interface for control change events.
+ */
+ public interface ControlChangeCallback {
+ /**
+ * Called whenever the control changes from the point of view of the
+ * registry. The callee can check whether they have control or not
+ * using the hasControl parameter.
+ *
+ * @param dpid The switch that control has changed for
+ * @param hasControl Whether the listener now has control or not
+ */
+ void controlChanged(long dpid, boolean hasControl);
+ }
+
+ /**
+ * Request for control of a switch. This method does not block. When
+ * control for a switch changes, the controlChanged method on the
+ * callback object will be called. This happens any time the control
+ * changes while the request is still active (until releaseControl is
+ * called)
+ *
+ * @param dpid Switch to request control for
+ * @param cb Callback that will be used to notify caller of control
+ * changes
+ * @throws RegistryException Errors contacting the registry service
+ */
+ public void requestControl(long dpid, ControlChangeCallback cb)
+ throws RegistryException;
+
+ /**
+ * Stop trying to take control of a switch. This removes the entry
+ * for this controller requesting this switch in the registry.
+ * If the controller had control when this is called, another controller
+ * will now gain control of the switch. This call doesn't block.
+ *
+ * @param dpid Switch to release control of
+ */
+ public void releaseControl(long dpid);
+
+ /**
+ * Check whether the controller has control of the switch
+ * This call doesn't block.
+ *
+ * @param dpid Switch to check control of
+ * @return true if controller has control of the switch.
+ */
+ public boolean hasControl(long dpid);
+
+ /**
+ * Check whether this instance is the leader for the cluster.
+ * This call doesn't block.
+ *
+ * @return true if the instance is the leader for the cluster,
+ * otherwise false.
+ */
+ public boolean isClusterLeader();
+
+ /**
+ * Get the unique ID used to identify this controller in the cluster.
+ *
+ * @return controller ID.
+ */
+ public String getControllerId();
+
+ /**
+ * Register a controller to the ONOS cluster. Must be called before
+ * the registry can be used to take control of any switches.
+ *
+ * @param controllerId A unique string ID identifying this controller
+ * in the cluster
+ * @throws RegistryException for errors connecting to registry service,
+ * controllerId already registered
+ */
+ public void registerController(String controllerId)
+ throws RegistryException;
+
+ /**
+ * Get all controllers in the cluster.
+ *
+ * @return Collection of controller IDs
+ * @throws RegistryException on error
+ */
+ public Collection<String> getAllControllers() throws RegistryException;
+
+ /**
+ * Get all switches in the cluster, along with which controller is
+ * in control of them (if any) and any other controllers that have
+ * requested control.
+ *
+ * @return Map of all switches.
+ */
+ public Map<String, List<ControllerRegistryEntry>> getAllSwitches();
+
+ /**
+ * Get the controller that has control of a given switch.
+ *
+ * @param dpid Switch to find controller for
+ * @return controller ID
+ * @throws RegistryException Errors contacting registry service
+ */
+ public String getControllerForSwitch(long dpid) throws RegistryException;
+
+ /**
+ * Get all switches controlled by a given controller.
+ *
+ * @param controllerId ID of the controller
+ * @return Collection of dpids
+ */
+ public Collection<Long> getSwitchesControlledByController(String controllerId);
+
+ /**
+ * Get a unique Id Block.
+ *
+ * @return Id Block.
+ */
+ public IdBlock allocateUniqueIdBlock();
+
+ /**
+ * Get next unique id and retrieve a new range of ids if needed.
+ * @param range range to use for the identifier
+ * @return Id Block.
+ */
+ public IdBlock allocateUniqueIdBlock(long range);
+
+
+ /**
+ * Get a globally unique ID.
+ *
+ * @return a globally unique ID.
+ */
+ public long getNextUniqueId();
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/IdBlock.java b/src/main/java/net/onrc/onos/core/registry/IdBlock.java
new file mode 100644
index 0000000..deb02bf
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/IdBlock.java
@@ -0,0 +1,31 @@
+package net.onrc.onos.core.registry;
+
+public class IdBlock {
+ private long start;
+ private long end;
+ private long size;
+
+ public IdBlock(long start, long end, long size) {
+ this.start = start;
+ this.end = end;
+ this.size = size;
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public long getEnd() {
+ return end;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public String toString() {
+ return "IdBlock [start=" + start + ", end=" + end + ", size=" + size
+ + "]";
+ }
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/RegistryException.java b/src/main/java/net/onrc/onos/core/registry/RegistryException.java
new file mode 100644
index 0000000..9bf0876
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/RegistryException.java
@@ -0,0 +1,26 @@
+package net.onrc.onos.core.registry;
+
+public class RegistryException extends Exception {
+
+ private static final long serialVersionUID = -8276300722010217913L;
+
+ /*
+ public RegistryException() {
+ // TODO Auto-generated constructor stub
+ }
+
+ public RegistryException(Throwable cause) {
+ super(cause);
+ // TODO Auto-generated constructor stub
+ }
+ */
+
+ public RegistryException(String message) {
+ super(message);
+ }
+
+ public RegistryException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/StandaloneRegistry.java b/src/main/java/net/onrc/onos/core/registry/StandaloneRegistry.java
new file mode 100755
index 0000000..88125ed
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/StandaloneRegistry.java
@@ -0,0 +1,202 @@
+package net.onrc.onos.core.registry;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import net.floodlightcontroller.core.IFloodlightProviderService;
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
+import net.floodlightcontroller.core.module.IFloodlightModule;
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.restserver.IRestApiService;
+import net.onrc.onos.core.registry.web.RegistryWebRoutable;
+
+import org.openflow.util.HexString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implementation of a registry that doesn't rely on any external registry
+ * service. This is designed to be used only in single-node setups (e.g. for
+ * development). All registry data is stored in local memory.
+ * @author jono
+ *
+ */
+public class StandaloneRegistry implements IFloodlightModule,
+ IControllerRegistryService {
+ protected final static Logger log = LoggerFactory.getLogger(StandaloneRegistry.class);
+
+ protected IRestApiService restApi;
+
+ protected String controllerId = null;
+ protected Map<String, ControlChangeCallback> switchCallbacks;
+
+ //
+ // Unique ID generation state
+ //
+ private static AtomicLong nextUniqueId = new AtomicLong(0);
+
+ @Override
+ public void requestControl(long dpid, ControlChangeCallback cb)
+ throws RegistryException {
+ if (controllerId == null) {
+ throw new RuntimeException(
+ "Must register a controller before calling requestControl");
+ }
+
+ switchCallbacks.put(HexString.toHexString(dpid), cb);
+
+ log.debug("Control granted for {}", HexString.toHexString(dpid));
+
+ //Immediately grant request for control
+ if (cb != null) {
+ cb.controlChanged(dpid, true);
+ }
+ }
+
+ @Override
+ public void releaseControl(long dpid) {
+ ControlChangeCallback cb = switchCallbacks.remove(HexString.toHexString(dpid));
+
+ log.debug("Control released for {}", HexString.toHexString(dpid));
+
+ if (cb != null){
+ cb.controlChanged(dpid, false);
+ }
+ }
+
+ @Override
+ public boolean hasControl(long dpid) {
+ return switchCallbacks.containsKey(HexString.toHexString(dpid));
+ }
+
+ @Override
+ public boolean isClusterLeader() {
+ return true;
+ }
+
+ @Override
+ public String getControllerId() {
+ return controllerId;
+ }
+
+ @Override
+ public void registerController(String controllerId)
+ throws RegistryException {
+ if (this.controllerId != null) {
+ throw new RegistryException(
+ "Controller already registered with id " + this.controllerId);
+ }
+ this.controllerId = controllerId;
+ }
+
+ @Override
+ public Collection<String> getAllControllers() throws RegistryException {
+ List<String> l = new ArrayList<String>();
+ l.add(controllerId);
+ return l;
+ }
+
+ @Override
+ public String getControllerForSwitch(long dpid) throws RegistryException {
+ return (switchCallbacks.get(HexString.toHexString(dpid)) != null)? controllerId: null;
+ }
+
+ @Override
+ public Map<String, List<ControllerRegistryEntry>> getAllSwitches() {
+ Map<String, List<ControllerRegistryEntry>> switches =
+ new HashMap<String, List<ControllerRegistryEntry>>();
+
+ for (String strSwitch : switchCallbacks.keySet()){
+ log.debug("Switch _{}", strSwitch);
+ List<ControllerRegistryEntry> list = new ArrayList<ControllerRegistryEntry>();
+ list.add(new ControllerRegistryEntry(controllerId, 0));
+
+ switches.put(strSwitch, list);
+ }
+
+ return switches;
+ }
+
+ @Override
+ public Collection<Long> getSwitchesControlledByController(
+ String controllerId) {
+ throw new RuntimeException("Not yet implemented");
+ }
+
+ private long blockTop = 0L;
+ private static final long BLOCK_SIZE = 0x1000000L;
+
+ /**
+ * Returns a block of IDs which are unique and unused.
+ * Range of IDs is fixed size and is assigned incrementally as this method called.
+ */
+ @Override
+ public synchronized IdBlock allocateUniqueIdBlock(){
+ long blockHead = blockTop;
+ long blockTail = blockTop + BLOCK_SIZE;
+
+ IdBlock block = new IdBlock(blockHead, blockTail - 1, BLOCK_SIZE);
+ blockTop = blockTail;
+
+ return block;
+ }
+
+ /**
+ * Get a globally unique ID.
+ *
+ * @return a globally unique ID.
+ */
+ @Override
+ public long getNextUniqueId() {
+ return nextUniqueId.incrementAndGet();
+ }
+
+ @Override
+ public Collection<Class<? extends IFloodlightService>> getModuleServices() {
+ Collection<Class<? extends IFloodlightService>> l =
+ new ArrayList<Class<? extends IFloodlightService>>();
+ l.add(IControllerRegistryService.class);
+ return l;
+ }
+
+ @Override
+ public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() {
+ Map<Class<? extends IFloodlightService>, IFloodlightService> m =
+ new HashMap<Class<? extends IFloodlightService>, IFloodlightService>();
+ m.put(IControllerRegistryService.class, this);
+ return m;
+ }
+
+ @Override
+ public Collection<Class<? extends IFloodlightService>> getModuleDependencies() {
+ Collection<Class<? extends IFloodlightService>> l =
+ new ArrayList<Class<? extends IFloodlightService>>();
+ l.add(IFloodlightProviderService.class);
+ l.add(IRestApiService.class);
+ return l;
+ }
+
+ @Override
+ public void init(FloodlightModuleContext context)
+ throws FloodlightModuleException {
+ restApi = context.getServiceImpl(IRestApiService.class);
+
+ switchCallbacks = new HashMap<String, ControlChangeCallback>();
+ }
+
+ @Override
+ public void startUp(FloodlightModuleContext context) {
+ restApi.addRestletRoutable(new RegistryWebRoutable());
+ }
+
+ @Override
+ public IdBlock allocateUniqueIdBlock(long range) {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/SwitchLeadershipData.java b/src/main/java/net/onrc/onos/core/registry/SwitchLeadershipData.java
new file mode 100644
index 0000000..9ccd47b
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/SwitchLeadershipData.java
@@ -0,0 +1,33 @@
+package net.onrc.onos.core.registry;
+
+import net.onrc.onos.core.registry.IControllerRegistryService.ControlChangeCallback;
+
+import org.apache.curator.framework.recipes.leader.LeaderLatch;
+import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
+
+public class SwitchLeadershipData {
+
+ private final LeaderLatch latch;
+ private final ControlChangeCallback cb;
+ private final LeaderLatchListener listener;
+
+ public SwitchLeadershipData(LeaderLatch latch, ControlChangeCallback cb,
+ LeaderLatchListener listener) {
+ this.latch = latch;
+ this.cb = cb;
+ this.listener = listener;
+ }
+
+ public LeaderLatch getLatch(){
+ return latch;
+ }
+
+ public ControlChangeCallback getCallback(){
+ return cb;
+ }
+
+ public LeaderLatchListener getListener() {
+ return listener;
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/ZookeeperRegistry.java b/src/main/java/net/onrc/onos/core/registry/ZookeeperRegistry.java
new file mode 100755
index 0000000..aa3e98b
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/ZookeeperRegistry.java
@@ -0,0 +1,650 @@
+package net.onrc.onos.core.registry;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import net.floodlightcontroller.core.IFloodlightProviderService;
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
+import net.floodlightcontroller.core.module.IFloodlightModule;
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.restserver.IRestApiService;
+import net.onrc.onos.core.registry.web.RegistryWebRoutable;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.recipes.atomic.AtomicValue;
+import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.curator.framework.recipes.leader.LeaderLatch;
+import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
+import org.apache.curator.framework.recipes.leader.Participant;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.retry.RetryOneTime;
+import org.apache.curator.x.discovery.ServiceCache;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.openflow.util.HexString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+
+/**
+ * A registry service that uses Zookeeper. All data is stored in Zookeeper,
+ * so this can be used as a global registry in a multi-node ONOS cluster.
+ * @author jono
+ *
+ */
+public class ZookeeperRegistry implements IFloodlightModule, IControllerRegistryService {
+
+ protected final static Logger log = LoggerFactory.getLogger(ZookeeperRegistry.class);
+ protected String controllerId = null;
+
+ protected IRestApiService restApi;
+
+ //This is the default, it's overwritten by the connectionString configuration parameter
+ protected String connectionString = "localhost:2181";
+
+ private final String namespace = "onos";
+ private final String switchLatchesPath = "/switches";
+ private final String CLUSTER_LEADER_PATH = "/cluster/leader";
+
+ private final String SERVICES_PATH = "/"; //i.e. the root of our namespace
+ private final String CONTROLLER_SERVICE_NAME = "controllers";
+
+ protected CuratorFramework client;
+
+ protected PathChildrenCache switchCache;
+
+ protected ConcurrentHashMap<String, SwitchLeadershipData> switches;
+ protected Map<String, PathChildrenCache> switchPathCaches;
+
+ protected LeaderLatch clusterLeaderLatch;
+ protected ClusterLeaderListener clusterLeaderListener;
+ private static final long CLUSTER_LEADER_ELECTION_RETRY_MS = 100;
+
+ private final String ID_COUNTER_PATH = "/flowidcounter";
+ private final Long ID_BLOCK_SIZE = 0x100000000L;
+ protected DistributedAtomicLong distributedIdCounter;
+
+ //Zookeeper performance-related configuration
+ protected static final int sessionTimeout = 5000;
+ protected static final int connectionTimeout = 7000;
+
+ //
+ // Unique ID generation state
+ // TODO: The implementation must be updated to use the Zookeeper
+ // instead of a ramdon generator.
+ //
+ private static Random randomGenerator = new Random();
+ private static int nextUniqueIdPrefix = 0;
+ private static int nextUniqueIdSuffix = 0;
+
+ private final BlockingQueue<SwitchLeaderEvent> switchLeadershipEvents =
+ new LinkedBlockingQueue<SwitchLeaderEvent>();
+
+ private ExecutorService eventThreadExecutorService;
+
+ private static class SwitchLeaderEvent {
+ public final long dpid;
+ public final boolean isLeader;
+
+ public SwitchLeaderEvent(long dpid, boolean isLeader) {
+ this.dpid = dpid;
+ this.isLeader = isLeader;
+ }
+ }
+
+ /*
+ * Dispatcher thread for leadership change events coming from Curator.
+ */
+ private void dispatchEvents() {
+ while (!Thread.currentThread().isInterrupted()) {
+ try {
+ SwitchLeaderEvent event = switchLeadershipEvents.take();
+ SwitchLeadershipData swData = switches.get(HexString.toHexString(event.dpid));
+ if (swData == null) {
+ log.debug("Leadership data {} not found", event.dpid);
+ continue;
+ }
+
+ swData.getCallback().controlChanged(event.dpid, event.isLeader);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ } catch (Exception e) {
+ log.error("Exception in registry event thread", e);
+ }
+ }
+ }
+
+ protected class SwitchLeaderListener implements LeaderLatchListener{
+ String dpid;
+ LeaderLatch latch;
+
+ public SwitchLeaderListener(String dpid, LeaderLatch latch){
+ this.dpid = dpid;
+ this.latch = latch;
+ }
+
+ @Override
+ public void isLeader() {
+ log.debug("Became leader for {}", dpid);
+
+ switchLeadershipEvents.offer(new SwitchLeaderEvent(HexString.toLong(dpid), true));
+ }
+
+ @Override
+ public void notLeader() {
+ log.debug("Lost leadership for {}", dpid);
+
+ switchLeadershipEvents.offer(new SwitchLeaderEvent(HexString.toLong(dpid), false));
+ }
+ }
+
+ protected class SwitchPathCacheListener implements PathChildrenCacheListener {
+ @Override
+ public void childEvent(CuratorFramework client,
+ PathChildrenCacheEvent event) throws Exception {
+
+ String strSwitch = null;
+ if (event.getData() != null){
+ String[] splitted = event.getData().getPath().split("/");
+ strSwitch = splitted[splitted.length - 1];
+ }
+
+ switch (event.getType()){
+ case CHILD_ADDED:
+ case CHILD_UPDATED:
+ //Check we have a PathChildrenCache for this child, add one if not
+ synchronized (switchPathCaches){
+ if (switchPathCaches.get(strSwitch) == null){
+ PathChildrenCache pc = new PathChildrenCache(client,
+ event.getData().getPath(), true);
+ pc.start(StartMode.NORMAL);
+ switchPathCaches.put(strSwitch, pc);
+ }
+ }
+ break;
+ case CHILD_REMOVED:
+ //Remove our PathChildrenCache for this child
+ PathChildrenCache pc = null;
+ synchronized(switchPathCaches){
+ pc = switchPathCaches.remove(strSwitch);
+ }
+ if (pc != null){
+ pc.close();
+ }
+ break;
+ default:
+ //All other switchLeadershipEvents are connection status switchLeadershipEvents. We don't need to
+ //do anything as the path cache handles these on its own.
+ break;
+ }
+
+ }
+ };
+
+ protected class ClusterLeaderListener implements LeaderLatchListener {
+ LeaderLatch latch;
+
+ public ClusterLeaderListener(LeaderLatch latch) {
+ this.latch = latch;
+ }
+
+ //
+ // NOTE: If we need to support callbacks when the
+ // leadership changes, those should be called here.
+ //
+
+ @Override
+ public void isLeader() {
+ log.debug("Cluster leadership aquired");
+ }
+
+ @Override
+ public void notLeader() {
+ log.debug("Cluster leadership lost");
+ }
+ }
+
+ /**
+ * Listens for changes to the switch znodes in Zookeeper. This maintains
+ * the second level of PathChildrenCaches that hold the controllers
+ * contending for each switch - there's one for each switch.
+ */
+ PathChildrenCacheListener switchPathCacheListener = new SwitchPathCacheListener();
+ protected ServiceDiscovery<ControllerService> serviceDiscovery;
+ protected ServiceCache<ControllerService> serviceCache;
+
+
+ @Override
+ public void requestControl(long dpid, ControlChangeCallback cb) throws RegistryException {
+ log.info("Requesting control for {}", HexString.toHexString(dpid));
+
+ if (controllerId == null){
+ throw new RuntimeException("Must register a controller before calling requestControl");
+ }
+
+ String dpidStr = HexString.toHexString(dpid);
+ String latchPath = switchLatchesPath + "/" + dpidStr;
+
+ if (switches.get(dpidStr) != null){
+ log.debug("Already contesting {}, returning", HexString.toHexString(dpid));
+ throw new RegistryException("Already contesting control for " + dpidStr);
+ }
+
+ LeaderLatch latch = new LeaderLatch(client, latchPath, controllerId);
+ SwitchLeaderListener listener = new SwitchLeaderListener(dpidStr, latch);
+ latch.addListener(listener);
+
+
+ SwitchLeadershipData swData = new SwitchLeadershipData(latch, cb, listener);
+ SwitchLeadershipData oldData = switches.putIfAbsent(dpidStr, swData);
+
+ if (oldData != null){
+ //There was already data for that key in the map
+ //i.e. someone else got here first so we can't succeed
+ log.debug("Already requested control for {}", dpidStr);
+ throw new RegistryException("Already requested control for " + dpidStr);
+ }
+
+ //Now that we know we were able to add our latch to the collection,
+ //we can start the leader election in Zookeeper. However I don't know
+ //how to handle if the start fails - the latch is already in our
+ //switches list.
+ //TODO seems like there's a Curator bug when latch.start is called when
+ //there's no Zookeeper connection which causes two znodes to be put in
+ //Zookeeper at the latch path when we reconnect to Zookeeper.
+ try {
+ latch.start();
+ } catch (Exception e) {
+ log.warn("Error starting leader latch: {}", e.getMessage());
+ throw new RegistryException("Error starting leader latch for " + dpidStr, e);
+ }
+
+ }
+
+ @Override
+ public void releaseControl(long dpid) {
+ log.info("Releasing control for {}", HexString.toHexString(dpid));
+
+ String dpidStr = HexString.toHexString(dpid);
+
+ SwitchLeadershipData swData = switches.remove(dpidStr);
+
+ if (swData == null) {
+ log.debug("Trying to release control of a switch we are not contesting");
+ return;
+ }
+
+ LeaderLatch latch = swData.getLatch();
+
+ latch.removeListener(swData.getListener());
+
+ try {
+ latch.close();
+ } catch (IOException e) {
+ //I think it's OK not to do anything here. Either the node got
+ //deleted correctly, or the connection went down and the node got deleted.
+ log.debug("releaseControl: caught IOException {}", dpidStr);
+ }
+ }
+
+ @Override
+ public boolean hasControl(long dpid) {
+ String dpidStr = HexString.toHexString(dpid);
+
+ SwitchLeadershipData swData = switches.get(dpidStr);
+
+ if (swData == null) {
+ log.warn("No leader latch for dpid {}", dpidStr);
+ return false;
+ }
+
+ return swData.getLatch().hasLeadership();
+ }
+
+ @Override
+ public boolean isClusterLeader() {
+ return clusterLeaderLatch.hasLeadership();
+ }
+
+ @Override
+ public String getControllerId() {
+ return controllerId;
+ }
+
+ @Override
+ public Collection<String> getAllControllers() throws RegistryException {
+ log.debug("Getting all controllers");
+
+ List<String> controllers = new ArrayList<String>();
+ for (ServiceInstance<ControllerService> instance : serviceCache.getInstances()){
+ String id = instance.getPayload().getControllerId();
+ if (!controllers.contains(id)){
+ controllers.add(id);
+ }
+ }
+
+ return controllers;
+ }
+
+ @Override
+ public void registerController(String id) throws RegistryException {
+ if (controllerId != null) {
+ throw new RegistryException(
+ "Controller already registered with id " + controllerId);
+ }
+
+ controllerId = id;
+
+ try {
+ ServiceInstance<ControllerService> thisInstance = ServiceInstance.<ControllerService>builder()
+ .name(CONTROLLER_SERVICE_NAME)
+ .payload(new ControllerService(controllerId))
+ //.port((int)(65535 * Math.random())) // in a real application, you'd use a common port
+ //.uriSpec(uriSpec)
+ .build();
+
+ serviceDiscovery.registerService(thisInstance);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ }
+
+ @Override
+ public String getControllerForSwitch(long dpid) throws RegistryException {
+ String dpidStr = HexString.toHexString(dpid);
+
+ PathChildrenCache switchCache = switchPathCaches.get(dpidStr);
+
+ if (switchCache == null){
+ log.warn("Tried to get controller for non-existent switch");
+ return null;
+ }
+
+ try {
+ //We've seen issues with these caches get stuck out of date, so we'll have to
+ //force them to refresh before each read. This slows down the method as it
+ //blocks on a Zookeeper query, however at the moment only the cleanup thread
+ //uses this and that isn't particularly time-sensitive.
+ switchCache.rebuild();
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ List<ChildData> sortedData = new ArrayList<ChildData>(switchCache.getCurrentData());
+
+ Collections.sort(
+ sortedData,
+ new Comparator<ChildData>(){
+ private String getSequenceNumber(String path){
+ return path.substring(path.lastIndexOf('-') + 1);
+ }
+ @Override
+ public int compare(ChildData lhs, ChildData rhs) {
+ return getSequenceNumber(lhs.getPath()).
+ compareTo(getSequenceNumber(rhs.getPath()));
+ }
+ }
+ );
+
+ if (sortedData.size() == 0){
+ return null;
+ }
+
+ return new String(sortedData.get(0).getData(), Charsets.UTF_8);
+ }
+
+ @Override
+ public Collection<Long> getSwitchesControlledByController(String controllerId) {
+ //TODO remove this if not needed
+ throw new RuntimeException("Not yet implemented");
+ }
+
+
+ //TODO what should happen when there's no ZK connection? Currently we just return
+ //the cache but this may lead to false impressions - i.e. we don't actually know
+ //what's in ZK so we shouldn't say we do
+ @Override
+ public Map<String, List<ControllerRegistryEntry>> getAllSwitches() {
+ Map<String, List<ControllerRegistryEntry>> data =
+ new HashMap<String, List<ControllerRegistryEntry>>();
+
+ for (Map.Entry<String, PathChildrenCache> entry : switchPathCaches.entrySet()){
+ List<ControllerRegistryEntry> contendingControllers =
+ new ArrayList<ControllerRegistryEntry>();
+
+ if (entry.getValue().getCurrentData().size() < 1){
+ //TODO prevent even having the PathChildrenCache in this case
+ //log.info("Switch entry with no leader elections: {}", entry.getKey());
+ continue;
+ }
+
+ for (ChildData d : entry.getValue().getCurrentData()) {
+
+ String controllerId = new String(d.getData(), Charsets.UTF_8);
+
+ String[] splitted = d.getPath().split("-");
+ int sequenceNumber = Integer.parseInt(splitted[splitted.length - 1]);
+
+ contendingControllers.add(new ControllerRegistryEntry(controllerId, sequenceNumber));
+ }
+
+ Collections.sort(contendingControllers);
+ data.put(entry.getKey(), contendingControllers);
+ }
+ return data;
+ }
+
+ public IdBlock allocateUniqueIdBlock(long range) {
+ try {
+ AtomicValue<Long> result = null;
+ do {
+ result = distributedIdCounter.add(range);
+ } while (result == null || !result.succeeded());
+
+ return new IdBlock(result.preValue(), result.postValue() - 1, range);
+ } catch (Exception e) {
+ log.error("Error allocating ID block");
+ }
+ return null;
+ }
+
+ /**
+ * Returns a block of IDs which are unique and unused.
+ * Range of IDs is fixed size and is assigned incrementally as this method called.
+ * Since the range of IDs is managed by Zookeeper in distributed way, this method may block when
+ * requests come up simultaneously.
+ */
+ @Override
+ public IdBlock allocateUniqueIdBlock(){
+ return allocateUniqueIdBlock(ID_BLOCK_SIZE);
+ }
+
+ /**
+ * Get a globally unique ID.
+ *
+ * @return a globally unique ID.
+ */
+ @Override
+ public synchronized long getNextUniqueId() {
+ //
+ // Generate the next Unique ID.
+ //
+ // TODO: For now, the higher 32 bits are random, and
+ // the lower 32 bits are sequential.
+ // The implementation must be updated to use the Zookeeper
+ // to allocate the higher 32 bits (globally unique).
+ //
+ if ((nextUniqueIdSuffix & 0xffffffffL) == 0xffffffffL) {
+ nextUniqueIdPrefix = randomGenerator.nextInt();
+ nextUniqueIdSuffix = 0;
+ } else {
+ nextUniqueIdSuffix++;
+ }
+ long result = (long)nextUniqueIdPrefix << 32;
+ result = result | (0xffffffffL & nextUniqueIdSuffix);
+ return result;
+ }
+
+ /*
+ * IFloodlightModule
+ */
+
+ @Override
+ public Collection<Class<? extends IFloodlightService>> getModuleServices() {
+ Collection<Class<? extends IFloodlightService>> l =
+ new ArrayList<Class<? extends IFloodlightService>>();
+ l.add(IControllerRegistryService.class);
+ return l;
+ }
+
+ @Override
+ public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() {
+ Map<Class<? extends IFloodlightService>, IFloodlightService> m =
+ new HashMap<Class<? extends IFloodlightService>, IFloodlightService>();
+ m.put(IControllerRegistryService.class, this);
+ return m;
+ }
+
+ @Override
+ public Collection<Class<? extends IFloodlightService>> getModuleDependencies() {
+ Collection<Class<? extends IFloodlightService>> l =
+ new ArrayList<Class<? extends IFloodlightService>>();
+ l.add(IFloodlightProviderService.class);
+ l.add(IRestApiService.class);
+ return l;
+ }
+
+ //TODO currently blocks startup when it can't get a Zookeeper connection.
+ //Do we support starting up with no Zookeeper connection?
+ @Override
+ public void init (FloodlightModuleContext context) throws FloodlightModuleException {
+ log.info("Initialising the Zookeeper Registry - Zookeeper connection required");
+
+ //Read the Zookeeper connection string from the config
+ Map<String, String> configParams = context.getConfigParams(this);
+ String connectionString = configParams.get("connectionString");
+ if (connectionString != null){
+ this.connectionString = connectionString;
+ }
+ log.info("Setting Zookeeper connection string to {}", this.connectionString);
+
+ //
+ // Initialize the Unique ID generator
+ // TODO: This must be replaced by Zookeeper-based allocation
+ //
+ nextUniqueIdPrefix = randomGenerator.nextInt();
+
+ restApi = context.getServiceImpl(IRestApiService.class);
+
+ switches = new ConcurrentHashMap<String, SwitchLeadershipData>();
+ //switchPathCaches = new HashMap<String, PathChildrenCache>();
+ switchPathCaches = new ConcurrentHashMap<String, PathChildrenCache>();
+
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ client = CuratorFrameworkFactory.newClient(this.connectionString,
+ sessionTimeout, connectionTimeout, retryPolicy);
+
+ client.start();
+ client = client.usingNamespace(namespace);
+
+ distributedIdCounter = new DistributedAtomicLong(
+ client,
+ ID_COUNTER_PATH,
+ new RetryOneTime(100));
+
+ switchCache = new PathChildrenCache(client, switchLatchesPath, true);
+ switchCache.getListenable().addListener(switchPathCacheListener);
+
+ //Build the service discovery object
+ serviceDiscovery = ServiceDiscoveryBuilder.builder(ControllerService.class)
+ .client(client).basePath(SERVICES_PATH).build();
+
+ //We read the list of services very frequently (GUI periodically queries them)
+ //so we'll cache them to cut down on Zookeeper queries.
+ serviceCache = serviceDiscovery.serviceCacheBuilder()
+ .name(CONTROLLER_SERVICE_NAME).build();
+
+
+ try {
+ serviceDiscovery.start();
+ serviceCache.start();
+
+ //Don't prime the cache, we want a notification for each child node in the path
+ switchCache.start(StartMode.NORMAL);
+ } catch (Exception e) {
+ throw new FloodlightModuleException("Error initialising ZookeeperRegistry: "
+ + e.getMessage());
+ }
+
+ eventThreadExecutorService = Executors.newSingleThreadExecutor();
+ eventThreadExecutorService.execute(
+ new Runnable() {
+ @Override
+ public void run(){
+ dispatchEvents();
+ }
+ });
+ }
+
+ @Override
+ public void startUp (FloodlightModuleContext context) {
+ //
+ // Cluster Leader election setup.
+ // NOTE: We have to do it here, because during the init stage
+ // we don't know the Controller ID.
+ //
+ if (controllerId == null) {
+ log.error("Error on startup: unknown ControllerId");
+ }
+ clusterLeaderLatch = new LeaderLatch(client,
+ CLUSTER_LEADER_PATH,
+ controllerId);
+ clusterLeaderListener = new ClusterLeaderListener(clusterLeaderLatch);
+ clusterLeaderLatch.addListener(clusterLeaderListener);
+ try {
+ clusterLeaderLatch.start();
+ } catch (Exception e) {
+ log.error("Error on startup starting the cluster leader election: {}", e.getMessage());
+ }
+
+ // Keep trying until there is a cluster leader
+ do {
+ try {
+ Participant leader = clusterLeaderLatch.getLeader();
+ if (! leader.getId().isEmpty())
+ break;
+ Thread.sleep(CLUSTER_LEADER_ELECTION_RETRY_MS);
+ } catch (Exception e) {
+ log.error("Error on startup waiting for cluster leader election: {}", e.getMessage());
+ }
+ } while (true);
+
+ restApi.addRestletRoutable(new RegistryWebRoutable());
+ }
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/web/ControllerRegistryResource.java b/src/main/java/net/onrc/onos/core/registry/web/ControllerRegistryResource.java
new file mode 100644
index 0000000..94e7e62
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/web/ControllerRegistryResource.java
@@ -0,0 +1,38 @@
+package net.onrc.onos.core.registry.web;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import net.onrc.onos.core.registry.IControllerRegistryService;
+import net.onrc.onos.core.registry.RegistryException;
+
+import org.restlet.resource.Get;
+import org.restlet.resource.ServerResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ControllerRegistryResource extends ServerResource {
+
+ protected final static Logger log = LoggerFactory.getLogger(ControllerRegistryResource.class);
+
+ @Get("json")
+ public Collection<String> getControllers() {
+ IControllerRegistryService registry =
+ (IControllerRegistryService) getContext().getAttributes().
+ get(IControllerRegistryService.class.getCanonicalName());
+
+ Collection<String> controllers = null;
+ try {
+ controllers = registry.getAllControllers();
+ } catch (RegistryException e) {
+ log.warn("Error retrieving controller list: {}", e.getMessage());
+ }
+
+ if (controllers == null){
+ controllers = new ArrayList<String>();
+ }
+
+ return controllers;
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/web/RegistryWebRoutable.java b/src/main/java/net/onrc/onos/core/registry/web/RegistryWebRoutable.java
new file mode 100644
index 0000000..e6bf552
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/web/RegistryWebRoutable.java
@@ -0,0 +1,24 @@
+package net.onrc.onos.core.registry.web;
+
+import net.floodlightcontroller.restserver.RestletRoutable;
+
+import org.restlet.Context;
+import org.restlet.Restlet;
+import org.restlet.routing.Router;
+
+public class RegistryWebRoutable implements RestletRoutable {
+
+ @Override
+ public Restlet getRestlet(Context context) {
+ Router router = new Router(context);
+ router.attach("/controllers/json", ControllerRegistryResource.class);
+ router.attach("/switches/json", SwitchRegistryResource.class);
+ return router;
+ }
+
+ @Override
+ public String basePath() {
+ return "/wm/onos/registry";
+ }
+
+}
diff --git a/src/main/java/net/onrc/onos/core/registry/web/SwitchRegistryResource.java b/src/main/java/net/onrc/onos/core/registry/web/SwitchRegistryResource.java
new file mode 100644
index 0000000..60c4798
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/registry/web/SwitchRegistryResource.java
@@ -0,0 +1,30 @@
+package net.onrc.onos.core.registry.web;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.onrc.onos.core.registry.ControllerRegistryEntry;
+import net.onrc.onos.core.registry.IControllerRegistryService;
+
+import org.restlet.resource.Get;
+import org.restlet.resource.ServerResource;
+
+public class SwitchRegistryResource extends ServerResource {
+
+ @Get("json")
+ public Map<String, List<ControllerRegistryEntry>> getAllControllers(){
+ IControllerRegistryService registry =
+ (IControllerRegistryService) getContext().getAttributes().
+ get(IControllerRegistryService.class.getCanonicalName());
+
+ Map<String, List<ControllerRegistryEntry>> switches = null;
+ switches = registry.getAllSwitches();
+
+ if (switches == null){
+ switches = new HashMap<String, List<ControllerRegistryEntry>>();
+ }
+
+ return switches;
+ }
+}