Merged role changer code into master
diff --git a/mastership-test.sh b/mastership-test.sh
index 10474cf..209ffb7 100755
--- a/mastership-test.sh
+++ b/mastership-test.sh
@@ -1,2 +1,2 @@
#java -Dlogback.configurationFile=logback.xml -cp target/floodlight-only.jar:lib/*:lib/titan/* net.onrc.onos.registry.controller.RegistryRunner $1
-java -cp target/floodlight-only.jar:lib/*:lib/titan/* net.floodlightcontroller.core.Main -cf onos.properties
+java -Dlogback.configurationFile=logback.xml -cp target/floodlight-only.jar:lib/*:lib/titan/* net.floodlightcontroller.core.Main -cf onos.properties
diff --git a/src/main/java/net/floodlightcontroller/core/internal/Controller.java b/src/main/java/net/floodlightcontroller/core/internal/Controller.java
index 78b93ed..20ad566 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/Controller.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/Controller.java
@@ -22,8 +22,9 @@
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
-import java.util.ArrayList;
+import java.net.UnknownHostException;
import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@@ -52,13 +53,13 @@
import net.floodlightcontroller.core.IFloodlightProviderService;
import net.floodlightcontroller.core.IHAListener;
import net.floodlightcontroller.core.IInfoProvider;
-import net.floodlightcontroller.core.INetMapStorage.DM_OPERATION;
-import net.floodlightcontroller.core.IOFMessageListener;
import net.floodlightcontroller.core.IListener.Command;
+import net.floodlightcontroller.core.INetMapStorage.DM_OPERATION;
+import net.floodlightcontroller.core.INetMapTopologyService.ITopoRouteService;
+import net.floodlightcontroller.core.IOFMessageListener;
import net.floodlightcontroller.core.IOFSwitch;
import net.floodlightcontroller.core.IOFSwitchFilter;
import net.floodlightcontroller.core.IOFSwitchListener;
-import net.floodlightcontroller.core.INetMapTopologyService.ITopoRouteService;
import net.floodlightcontroller.core.ISwitchStorage.SwitchState;
import net.floodlightcontroller.core.annotations.LogMessageDoc;
import net.floodlightcontroller.core.annotations.LogMessageDocs;
@@ -77,6 +78,8 @@
import net.floodlightcontroller.storage.StorageException;
import net.floodlightcontroller.threadpool.IThreadPoolService;
import net.onrc.onos.registry.controller.IControllerRegistryService;
+import net.onrc.onos.registry.controller.IControllerRegistryService.ControlChangeCallback;
+import net.onrc.onos.registry.controller.RegistryException;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.buffer.ChannelBuffer;
@@ -109,9 +112,9 @@
import org.openflow.protocol.OFMessage;
import org.openflow.protocol.OFPacketIn;
import org.openflow.protocol.OFPhysicalPort;
-import org.openflow.protocol.OFPortStatus;
import org.openflow.protocol.OFPhysicalPort.OFPortConfig;
import org.openflow.protocol.OFPhysicalPort.OFPortState;
+import org.openflow.protocol.OFPortStatus;
import org.openflow.protocol.OFPortStatus.OFPortReason;
import org.openflow.protocol.OFSetConfig;
import org.openflow.protocol.OFStatisticsRequest;
@@ -193,7 +196,7 @@
protected IThreadPoolService threadPool;
protected IFlowService flowService;
protected ITopoRouteService topoRouteService;
- protected IControllerRegistryService masterHelper;
+ protected IControllerRegistryService registryService;
// Configuration options
protected int openFlowPort = 6633;
@@ -405,7 +408,7 @@
}
public void setMastershipService(IControllerRegistryService serviceImpl) {
- this.masterHelper = serviceImpl;
+ this.registryService = serviceImpl;
}
@Override
@@ -464,6 +467,63 @@
return new OFChannelHandler(state);
}
+ protected class RoleChangeCallback implements ControlChangeCallback {
+ @Override
+ public void controlChanged(long dpid, boolean hasControl) {
+ log.info("Role change callback for switch {}, hasControl {}",
+ HexString.toHexString(dpid), hasControl);
+
+ synchronized(roleChanger){
+ OFSwitchImpl sw = null;
+ for (OFSwitchImpl connectedSw : connectedSwitches){
+ if (connectedSw.getId() == dpid){
+ sw = connectedSw;
+ break;
+ }
+ }
+ if (sw == null){
+ log.warn("Switch {} not found in connected switches",
+ HexString.toHexString(dpid));
+ return;
+ }
+
+ Role role = null;
+
+ if (sw.getRole() == null){
+ if (hasControl){
+ role = Role.MASTER;
+ }
+ else {
+ role = Role.SLAVE;
+ }
+ }
+ else if (hasControl && sw.getRole() == Role.SLAVE) {
+ // Send a MASTER role request to the switch.
+ // If this is the first role request,
+ // this is a probe that we'll use to determine if the switch
+ // actually supports the role request message. If it does we'll
+ // get back a role reply message. If it doesn't we'll get back an
+ // OFError message.
+ // If role is MASTER we will promote switch to active
+ // list when we receive the switch's role reply messages
+ role = Role.MASTER;
+ }
+ else if (!hasControl && sw.getRole() == Role.MASTER) {
+ //Send a SLAVE role request to the switch
+ role = Role.SLAVE;
+ }
+
+ if (role != null) {
+ log.debug("Sending role request {} msg to {}", role, sw);
+ Collection<OFSwitchImpl> swList = new ArrayList<OFSwitchImpl>(1);
+ swList.add(sw);
+ roleChanger.submitRequest(swList, role);
+ }
+ }
+
+ }
+ }
+
/**
* Channel handler deals with the switch connection and dispatches
* switch messages to the appropriate locations.
@@ -511,6 +571,7 @@
removeSwitch(sw);
}
synchronized(roleChanger) {
+ registryService.releaseControl(sw.getId());
connectedSwitches.remove(sw);
}
sw.setConnected(false);
@@ -763,6 +824,22 @@
connectedSwitches.add(sw);
if (role != null) {
+ //Put the switch in SLAVE mode until we know we have control
+ log.debug("Setting new switch {} to SLAVE", sw.getStringId());
+ Collection<OFSwitchImpl> swList = new ArrayList<OFSwitchImpl>(1);
+ swList.add(sw);
+ roleChanger.submitRequest(swList, Role.SLAVE);
+
+ //Request control of the switch from the global registry
+ try {
+ registryService.requestControl(sw.getId(),
+ new RoleChangeCallback());
+ } catch (RegistryException e) {
+ log.debug("Registry error: {}", e.getMessage());
+ }
+
+
+
// Send a role request if role support is enabled for the controller
// This is a probe that we'll use to determine if the switch
// actually supports the role request message. If it does we'll
@@ -770,12 +847,14 @@
// OFError message.
// If role is MASTER we will promote switch to active
// list when we receive the switch's role reply messages
+ /*
log.debug("This controller's role is {}, " +
"sending initial role request msg to {}",
role, sw);
Collection<OFSwitchImpl> swList = new ArrayList<OFSwitchImpl>(1);
swList.add(sw);
roleChanger.submitRequest(swList, role);
+ */
}
else {
// Role supported not enabled on controller (for now)
@@ -2057,6 +2136,16 @@
if (controllerId != null) {
this.controllerId = controllerId;
}
+ else {
+ //Try to get the hostname of the machine and use that for controller ID
+ try {
+ String hostname = java.net.InetAddress.getLocalHost().getHostName();
+ this.controllerId = hostname;
+ } catch (UnknownHostException e) {
+ // Can't get hostname, we'll just use the default
+ }
+ }
+
log.debug("ControllerId set to {}", this.controllerId);
}
@@ -2097,7 +2186,9 @@
this.factory = new BasicFactory();
this.providerMap = new HashMap<String, List<IInfoProvider>>();
setConfigParams(configParams);
- this.role = getInitialRole(configParams);
+ //this.role = getInitialRole(configParams);
+ //Set the controller's role to MASTER so it always tries to do role requests.
+ this.role = Role.MASTER;
this.roleChanger = new RoleChanger();
initVendorMessages();
this.systemStartTime = System.currentTimeMillis();
@@ -2112,6 +2203,12 @@
"that the system database has failed to start. " +
LogMessageDoc.CHECK_CONTROLLER)
public void startupComponents() {
+ try {
+ registryService.registerController(controllerId);
+ } catch (RegistryException e2) {
+ log.warn("Registry service error: {}", e2.getMessage());
+ }
+
// Create the table names we use
storageSource.createTable(CONTROLLER_TABLE_NAME, null);
storageSource.createTable(SWITCH_TABLE_NAME, null);
diff --git a/src/main/java/net/floodlightcontroller/core/module/FloodlightModuleLoader.java b/src/main/java/net/floodlightcontroller/core/module/FloodlightModuleLoader.java
index 45fe997..5561327 100644
--- a/src/main/java/net/floodlightcontroller/core/module/FloodlightModuleLoader.java
+++ b/src/main/java/net/floodlightcontroller/core/module/FloodlightModuleLoader.java
@@ -85,7 +85,7 @@
try {
m = moduleIter.next();
} catch (ServiceConfigurationError sce) {
- logger.debug("Could not find module");
+ logger.debug("Could not find module: {}", sce.getMessage());
//moduleIter.remove();
continue;
}
diff --git a/src/main/java/net/onrc/onos/registry/controller/IControllerRegistryService.java b/src/main/java/net/onrc/onos/registry/controller/IControllerRegistryService.java
index e924f6a..afd9dc6 100644
--- a/src/main/java/net/onrc/onos/registry/controller/IControllerRegistryService.java
+++ b/src/main/java/net/onrc/onos/registry/controller/IControllerRegistryService.java
@@ -8,42 +8,91 @@
public interface IControllerRegistryService extends IFloodlightService {
- // Callback for all mastership changes.
- // Change callback is called when mastership is acquired or released
+ /**
+ * Callback interface for control change events
+ *
+ */
public interface ControlChangeCallback {
+ /**
+ * Called whenever the control changes from the point of view of the
+ * registry. The callee can check whether they have control or not
+ * using the hasControl parameter.
+ * @param dpid The switch that control has changed for
+ * @param hasControl Whether the listener now has control or not
+ */
public void controlChanged(long dpid, boolean hasControl);
}
- // Acquire mastership for a switch.
- public void requestControl(long dpid, ControlChangeCallback cb) throws RegistryException;
+ /**
+ * Request for control of a switch. This method does not block. When
+ * control for a switch changes, the controlChanged method on the
+ * callback object will be called. This happens any time the control
+ * changes while the request is still active (until releaseControl is
+ * called)
+ * @param dpid Switch to request control for
+ * @param cb Callback that will be used to notify caller of control
+ * changes
+ * @throws RegistryException Errors contacting the registry service
+ */
+ public void requestControl(long dpid, ControlChangeCallback cb)
+ throws RegistryException;
- // Release mastership for a switch
+ /**
+ * Stop trying to take control of a switch. This removes the entry
+ * for this controller requesting this switch in the registry.
+ * If the controller had control when this is called, another controller
+ * will now gain control of the switch. This call doesn't block.
+ * @param dpid Switch to release control of
+ */
public void releaseControl(long dpid);
-
- // Check if I am the master of a switch. This is a nonblocking call that checks if the caller is a
+
+ /**
+ * Check whether the controller has control of the switch
+ * This call doesn't block.
+ * @param dpid Switch to check control of
+ * @return
+ */
public boolean hasControl(long dpid);
- // Set/Get mastership identifier.
- // This is typically a unique identifier of the controller that does not change across restarts
+
+ /**
+ * Superseded by registerController
+ * @param id
+ */
+ @Deprecated
public void setMastershipId (String id);
+
+ /**
+ * Get the unique ID used to identify this controller in the cluster
+ * @return
+ */
public String getMastershipId ();
/**
- * Register a controller to the ONOS cluster
- * @param controller A string identifying the controller
+ * Register a controller to the ONOS cluster. Must be called before
+ * the registry can be used to take control of any switches.
+ * @param controller A unique string ID identifying this controller
+ * in the cluster
+ * @throws errors connecting to registry service,
+ * controllerId already registered
*/
public void registerController(String controllerId) throws RegistryException;
/**
* Get all controllers in the cluster
- * @return
+ * @return Collection of controller IDs
*/
public Collection<String> getAllControllers() throws RegistryException;
+ /**
+ * Get all switches in the cluster, along with which controller is
+ * in control of them (if any) and any other controllers that have
+ * requested control.
+ * @return
+ */
+ public Map<String, List<ControllerRegistryEntry>> getAllSwitches();
public String getControllerForSwitch(long dpid) throws RegistryException;
- public Map<String, List<ControllerRegistryEntry>> getAllSwitches();
-
public Collection<Long> getSwitchesControlledByController(String controllerId);
}
diff --git a/src/main/java/net/onrc/onos/registry/controller/IMastershipHelper.java b/src/main/java/net/onrc/onos/registry/controller/IMastershipHelper.java
deleted file mode 100644
index 9e07d46..0000000
--- a/src/main/java/net/onrc/onos/registry/controller/IMastershipHelper.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package net.onrc.onos.registry.controller;
-
-public interface IMastershipHelper {
-
- // Callback for all mastership changes.
- // Change callback is called when mastership is acquired or released
- public interface MastershipCallback {
- public void changeCallback(long dpid);
- }
-
- // Set/get mastership identifier. This is used to set the unique identifier of the controller that is asking for mastership.
- // It needs to be set first before any mastership call can be made
- public void setMastershipId (String id);
- public String getMastershipId ();
-
- // Request mastership for a switch. Our request for mastership remains in a queue. If we win mastership, the callback
- // is called. This call is non-blocking and can be called from the packet processing context as well.
- public void requestMastership(long dpid, MastershipCallback cb);
-
- // Release mastership for a switch. If we are the master, then the mastership will be released and given to the next
- // controller who had requested mastership. If we are not the master our request for mastership will be
- // removed from the queue.
- public void releaseMastership(long dpid);
-
- // Check if I am the master of a switch and return true if I am the master.
- public boolean amMaster(long dpid);
-}
diff --git a/src/main/java/net/onrc/onos/registry/controller/MastershipHelper.java b/src/main/java/net/onrc/onos/registry/controller/MastershipHelper.java
deleted file mode 100644
index 2fcd002..0000000
--- a/src/main/java/net/onrc/onos/registry/controller/MastershipHelper.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package net.onrc.onos.registry.controller;
-
-public class MastershipHelper implements IMastershipHelper {
-
- @Override
- public void setMastershipId(String id) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public String getMastershipId() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public void requestMastership(long dpid, MastershipCallback cb) {
- // TODO Auto-generated method stub
- return;
- }
-
- @Override
- public void releaseMastership(long dpid) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public boolean amMaster(long dpid) {
- // TODO Auto-generated method stub
- return false;
- }
-
-}
diff --git a/src/main/java/net/onrc/onos/registry/controller/RegistryException.java b/src/main/java/net/onrc/onos/registry/controller/RegistryException.java
index 3b237c2..fbc68a44 100644
--- a/src/main/java/net/onrc/onos/registry/controller/RegistryException.java
+++ b/src/main/java/net/onrc/onos/registry/controller/RegistryException.java
@@ -9,18 +9,16 @@
// TODO Auto-generated constructor stub
}
-
- public RegistryException(String message) {
- super(message);
- // TODO Auto-generated constructor stub
- }
-
public RegistryException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
*/
+ public RegistryException(String message) {
+ super(message);
+ }
+
public RegistryException(String message, Throwable cause) {
super(message, cause);
}
diff --git a/src/main/java/net/onrc/onos/registry/controller/StandaloneRegistry.java b/src/main/java/net/onrc/onos/registry/controller/StandaloneRegistry.java
index e3181fe..8a468bc 100644
--- a/src/main/java/net/onrc/onos/registry/controller/StandaloneRegistry.java
+++ b/src/main/java/net/onrc/onos/registry/controller/StandaloneRegistry.java
@@ -32,7 +32,8 @@
public void requestControl(long dpid, ControlChangeCallback cb)
throws RegistryException {
if (controllerId == null) {
- throw new RuntimeException("Must register a controller before calling requestControl");
+ throw new RuntimeException(
+ "Must register a controller before calling requestControl");
}
switchCallbacks.put(HexString.toHexString(dpid), cb);
@@ -75,6 +76,10 @@
@Override
public void registerController(String controllerId)
throws RegistryException {
+ if (this.controllerId != null) {
+ throw new RegistryException(
+ "Controller already registered with id " + this.controllerId);
+ }
this.controllerId = controllerId;
}
diff --git a/src/main/java/net/onrc/onos/registry/controller/ZookeeperRegistry.java b/src/main/java/net/onrc/onos/registry/controller/ZookeeperRegistry.java
index 58b8bc7..468bb07 100644
--- a/src/main/java/net/onrc/onos/registry/controller/ZookeeperRegistry.java
+++ b/src/main/java/net/onrc/onos/registry/controller/ZookeeperRegistry.java
@@ -22,6 +22,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Charsets;
import com.netflix.curator.RetryPolicy;
import com.netflix.curator.framework.CuratorFramework;
import com.netflix.curator.framework.CuratorFrameworkFactory;
@@ -59,7 +60,9 @@
protected Map<String, ControlChangeCallback> switchCallbacks;
protected Map<String, PathChildrenCache> switchPathCaches;
- //protected boolean zookeeperEnabled = false;
+ //Zookeeper performance-related configuration
+ protected static final int sessionTimeout = 2000;
+ protected static final int connectionTimeout = 4000;
protected class ParamaterizedCuratorWatcher implements CuratorWatcher {
private String dpid;
@@ -132,7 +135,6 @@
@Override
public void childEvent(CuratorFramework client,
PathChildrenCacheEvent event) throws Exception {
- // TODO Auto-generated method stub
log.debug("Root switch path cache got {} event", event.getType());
String strSwitch = null;
@@ -171,14 +173,6 @@
@Override
public void requestControl(long dpid, ControlChangeCallback cb) throws RegistryException {
- /*
- if (!zookeeperEnabled) {
- //If zookeeper connection is disabled all control requests succeed immediately
- if (cb != null){
- cb.controlChanged(dpid, true);
- }
- return;
- }*/
if (controllerId == null){
throw new RuntimeException("Must register a controller before calling requestControl");
@@ -211,7 +205,6 @@
@Override
public void releaseControl(long dpid) {
- //if (!zookeeperEnabled) return;
String dpidStr = HexString.toHexString(dpid);
@@ -234,7 +227,6 @@
@Override
public boolean hasControl(long dpid) {
- //if (!zookeeperEnabled) return false;
LeaderLatch latch = switchLatches.get(HexString.toHexString(dpid));
@@ -264,8 +256,6 @@
@Override
public Collection<String> getAllControllers() throws RegistryException {
- //if (!zookeeperEnabled) return null;
-
log.debug("Getting all controllers");
List<String> controllers = new ArrayList<String>();
@@ -285,16 +275,14 @@
@Override
public void registerController(String id) throws RegistryException {
- //if (!zookeeperEnabled) return;
+ if (controllerId != null) {
+ throw new RegistryException(
+ "Controller already registered with id " + controllerId);
+ }
controllerId = id;
- byte bytes[] = null;
- try {
- bytes = id.getBytes("UTF-8");
- } catch (UnsupportedEncodingException e1) {
- throw new RegistryException("Error encoding string", e1);
- }
+ byte bytes[] = id.getBytes(Charsets.UTF_8);
String path = controllerPath + "/" + id;
@@ -311,12 +299,10 @@
@Override
public String getControllerForSwitch(long dpid) throws RegistryException {
- //if (!zookeeperEnabled) return null;
// TODO Work out how we should store this controller/switch data.
// The leader latch might be a index to the /controllers collections
// which holds more info on the controller (how to talk to it for example).
-
String strDpid = HexString.toHexString(dpid);
LeaderLatch latch = switchLatches.get(strDpid);
@@ -357,19 +343,8 @@
}
for (ChildData d : entry.getValue().getCurrentData()) {
- /*
- if (d.getPath().length() < 1){
- log.info("Switch entry with no leader elections: {}", d.getPath());
- continue;
- }
- */
-
- String controllerId = null;
- try {
- controllerId = new String(d.getData(), "UTF-8");
- } catch (UnsupportedEncodingException e) {
- log.warn("Encoding exception: {}", e.getMessage());
- }
+
+ String controllerId = new String(d.getData(), Charsets.UTF_8);
String[] splitted = d.getPath().split("-");
int sequenceNumber = Integer.parseInt(splitted[splitted.length - 1]);
@@ -413,56 +388,30 @@
@Override
public void init (FloodlightModuleContext context) throws FloodlightModuleException {
-
log.info("Initialising the Zookeeper Registry - Zookeeper connection required");
- restApi = context.getServiceImpl(IRestApiService.class);
-
- //We have a config option that determines whether we try and connect to
- //zookeeper or not. By default zookeeper connection is disabled. When we don't
- //have a zookeeper connection we act as though we are the only server in the
- //cluster, i.e. all control requests will succeed.
- /*Map<String, String> configOptions = context.getConfigParams(this);
- String enableZookeeper = configOptions.get("enableZookeeper");
- if (enableZookeeper != null) {
- log.info("Enabling Zookeeper connection");
- zookeeperEnabled = true;
+ //Read the Zookeeper connection string from the config
+ Map<String, String> configParams = context.getConfigParams(this);
+ String connectionString = configParams.get("connectionString");
+ if (connectionString != null){
+ this.connectionString = connectionString;
}
- else {
- log.info("Zookeeper connectivity is disabled - running in standalone mode");
- return;
- }*/
+ log.info("Setting Zookeeper connection string to {}", this.connectionString);
- /*
- try {
- String localHostname = java.net.InetAddress.getLocalHost().getHostName();
- controllerId = localHostname;
- log.debug("Setting controller id to {}", controllerId);
- } catch (UnknownHostException e) {
- // TODO Handle this exception
- e.printStackTrace();
- }*/
+ restApi = context.getServiceImpl(IRestApiService.class);
switchLatches = new HashMap<String, LeaderLatch>();
switchCallbacks = new HashMap<String, ControlChangeCallback>();
switchPathCaches = new HashMap<String, PathChildrenCache>();
RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- client = CuratorFrameworkFactory.newClient(connectionString, retryPolicy);
+ client = CuratorFrameworkFactory.newClient(this.connectionString,
+ sessionTimeout, connectionTimeout, retryPolicy);
client.start();
client = client.usingNamespace(namespace);
-
- //Put some data in for testing
- /*
- try {
- registerController("zookeeperController");
- requestControl(2L, null);
- } catch (RegistryException e1) {
- // TODO Auto-generated catch block
- e1.printStackTrace();
- }*/
+
controllerCache = new PathChildrenCache(client, controllerPath, true);
switchCache = new PathChildrenCache(client, switchLatchesPath, true);