diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/IOFSwitch.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/IOFSwitch.java
new file mode 100644
index 0000000..f274a28
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/IOFSwitch.java
@@ -0,0 +1,610 @@
+/**
+ *    Copyright 2011, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Future;
+
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounterService;
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounterService.CounterException;
+import net.onrc.onos.of.ctl.util.OrderedCollection;
+
+import org.jboss.netty.channel.Channel;
+import org.projectfloodlight.openflow.protocol.OFActionType;
+import org.projectfloodlight.openflow.protocol.OFCapabilities;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFFeaturesReply;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFPortStatus;
+import org.projectfloodlight.openflow.protocol.OFStatsReply;
+import org.projectfloodlight.openflow.protocol.OFStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.types.U64;
+
+
+public interface IOFSwitch {
+
+    /**
+     * OF1.3 switches should support role-request messages as in the 1.3 spec.
+     * OF1.0 switches may or may not support the Nicira role request extensions.
+     * To indicate the support, this property should be set by the associated
+     * OF1.0 switch driver in the net.onrc.onos.core.drivermanager package.
+     * The property will be ignored for OF1.3 switches.
+     */
+    public static final String SWITCH_SUPPORTS_NX_ROLE = "supportsNxRole";
+
+
+    //************************
+    // Channel related
+    //************************
+
+    /**
+     * Disconnects the switch by closing the TCP connection. Results in a call
+     * to the channel handler's channelDisconnected method for cleanup
+     * @throws IOException
+     */
+    public void disconnectSwitch();
+
+    /**
+     * Writes to the OFMessage to the output stream.
+     * The message will be handed to the floodlightProvider for possible filtering
+     * and processing by message listeners
+     *
+     * @param m
+     * @param bc
+     * @throws IOException
+     */
+    public void write(OFMessage m) throws IOException;
+
+    /**
+     * Writes the list of messages to the output stream.
+     * The message will be handed to the floodlightProvider for possible filtering
+     * and processing by message listeners.
+     *
+     * @param msglist
+     * @param bc
+     * @throws IOException
+     */
+    public void write(List<OFMessage> msglist) throws IOException;
+
+    /**
+     * Gets the date the switch connected to this controller.
+     *
+     * @return the date
+     */
+    public Date getConnectedSince();
+
+    /**
+     * Gets the next available transaction id.
+     *
+     * @return the next transaction ID
+     */
+    public int getNextTransactionId();
+
+    /**
+     * Checks if the switch is still connected.
+     * Only call while holding processMessageLock
+     *
+     * @return whether the switch is still disconnected
+     */
+    public boolean isConnected();
+
+    /**
+     * Sets whether the switch is connected.
+     * Only call while holding modifySwitchLock
+     *
+     * @param connected whether the switch is connected
+     */
+    public void setConnected(boolean connected);
+
+    /**
+     * Flushes all flows queued for this switch in the current thread.
+     * NOTE: The contract is limited to the current thread
+     */
+    public void flush();
+
+    /**
+     * Sets the Netty Channel this switch instance is associated with.
+     * <p>
+     * Called immediately after instantiation
+     *
+     * @param channel the channel
+     */
+    public void setChannel(Channel channel);
+
+    //************************
+    // Switch features related
+    //************************
+
+    /**
+     * Gets the datapathId of the switch.
+     *
+     * @return the switch buffers
+     */
+    public long getId();
+
+    /**
+     * Gets a string version of the ID for this switch.
+     *
+     * @return string version of the ID
+     */
+    public String getStringId();
+
+    /**
+     * Gets the number of buffers.
+     *
+     * @return the number of buffers
+     */
+    public int getNumBuffers();
+
+    public Set<OFCapabilities> getCapabilities();
+
+    public byte getNumTables();
+
+    /**
+     * Returns an OFDescStatsReply message object. Use the methods contained
+     * to retrieve switch descriptions for Manufacturer, Hw/Sw version etc.
+     */
+    public OFDescStatsReply getSwitchDescription();
+
+    /**
+     * Cancel features reply with a specific transaction ID.
+     * @param transactionId the transaction ID
+     */
+    public void cancelFeaturesReply(int transactionId);
+
+    /**
+     * Gets the OFActionType set.
+     * <p>
+     * getActions has relevance only for an OpenFlow 1.0 switch.
+     * For OF1.3, each table can support different actions
+     *
+     * @return the action set
+     */
+    public Set<OFActionType> getActions();
+
+    public void setOFVersion(OFVersion ofv);
+
+    public OFVersion getOFVersion();
+
+
+    //************************
+    //  Switch port related
+    //************************
+
+    /**
+     * the type of change that happened to an open flow port.
+     */
+    public enum PortChangeType {
+        /** Either a new port has been added by the switch, or we are
+         * adding a port we just deleted (via a prior notification) due to
+         * a change in the portNumber-portName mapping.
+         */
+        ADD,
+        /** some other feature of the port has changed (eg. speed)*/
+        OTHER_UPDATE,
+        /** Either a port has been deleted by the switch, or we are deleting
+         * a port whose portNumber-portName mapping has changed. Note that in
+         * the latter case, a subsequent notification will be sent out to add a
+         * port with the new portNumber-portName mapping.
+         */
+        DELETE,
+        /** Port is up (i.e. enabled). Presumably an earlier notification had
+         * indicated that it was down. To be UP implies that the port is
+         * administratively considered UP (see ofp_port_config) AND the port
+         * link is up AND the port is no longer blocked (see ofp_port_state).
+         */
+        UP,
+        /** Port is down (i.e. disabled). Presumably an earlier notification had
+         * indicated that it was up, or the port was always up.
+         * To be DOWN implies that the port has been either
+         * administratively brought down (see ofp_port_config) OR the port
+         * link is down OR the port is blocked (see ofp_port_state).
+         */
+        DOWN,
+    }
+
+    /**
+     * Describes a change of an open flow port.
+     */
+    public static class PortChangeEvent {
+        public final OFPortDesc port;
+        public final PortChangeType type;
+        /**
+         * @param port
+         * @param type
+         */
+        public PortChangeEvent(OFPortDesc port,
+                               PortChangeType type) {
+            this.port = port;
+            this.type = type;
+        }
+        /* (non-Javadoc)
+         * @see java.lang.Object#hashCode()
+         */
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + ((port == null) ? 0 : port.hashCode());
+            result = prime * result + ((type == null) ? 0 : type.hashCode());
+            return result;
+        }
+        /* (non-Javadoc)
+         * @see java.lang.Object#equals(java.lang.Object)
+         */
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            PortChangeEvent other = (PortChangeEvent) obj;
+            if (port == null) {
+                if (other.port != null) {
+                    return false;
+                }
+            } else if (!port.equals(other.port)) {
+                return false;
+            }
+            if (type != other.type) {
+                return false;
+            }
+            return true;
+        }
+        /* (non-Javadoc)
+         * @see java.lang.Object#toString()
+         */
+        @Override
+        public String toString() {
+            return "[" + type + " " + port.toString() + "]";
+        }
+    }
+
+
+    /**
+     * Get list of all enabled ports. This will typically be different from
+     * the list of ports in the OFFeaturesReply, since that one is a static
+     * snapshot of the ports at the time the switch connected to the controller
+     * whereas this port list also reflects the port status messages that have
+     * been received.
+     *
+     * @return Unmodifiable list of ports not backed by the underlying collection
+     */
+    public Collection<OFPortDesc> getEnabledPorts();
+
+    /**
+     * Get list of the port numbers of all enabled ports. This will typically
+     * be different from the list of ports in the OFFeaturesReply, since that
+     * one is a static snapshot of the ports at the time the switch connected
+     * to the controller whereas this port list also reflects the port status
+     * messages that have been received.
+     *
+     * @return Unmodifiable list of ports not backed by the underlying collection
+     */
+    public Collection<Integer> getEnabledPortNumbers();
+
+    /**
+     * Retrieve the port object by the port number. The port object
+     * is the one that reflects the port status updates that have been
+     * received, not the one from the features reply.
+     *
+     * @param portNumber
+     * @return port object
+     */
+    public OFPortDesc getPort(int portNumber);
+
+    /**
+     * Retrieve the port object by the port name. The port object
+     * is the one that reflects the port status updates that have been
+     * received, not the one from the features reply.
+     *
+     * @param portName
+     * @return port object
+     */
+    public OFPortDesc getPort(String portName);
+
+    /**
+     * Add or modify a switch port. This is called by the core controller
+     * code in response to a OFPortStatus message. It should not typically be
+     * called by other floodlight applications.
+     *
+     * OFPPR_MODIFY and OFPPR_ADD will be treated as equivalent. The OpenFlow
+     * spec is not clear on whether portNames are portNumbers are considered
+     * authoritative identifiers. We treat portNames <-> portNumber mappings
+     * as fixed. If they change, we delete all previous conflicting ports and
+     * add all new ports.
+     *
+     * @param ps the port status message
+     * @return the ordered Collection of changes "applied" to the old ports
+     * of the switch according to the PortStatus message. A single PortStatus
+     * message can result in multiple changes.
+     * If portName <-> portNumber mappings have
+     * changed, the iteration order ensures that delete events for old
+     * conflicting appear before before events adding new ports
+     */
+    public OrderedCollection<PortChangeEvent> processOFPortStatus(OFPortStatus ps);
+
+    /**
+     * Get list of all ports. This will typically be different from
+     * the list of ports in the OFFeaturesReply, since that one is a static
+     * snapshot of the ports at the time the switch connected to the controller
+     * whereas this port list also reflects the port status messages that have
+     * been received.
+     *
+     * @return Unmodifiable list of ports
+     */
+    public Collection<OFPortDesc> getPorts();
+
+    /**
+     * @param portName
+     * @return Whether a port is enabled per latest port status message
+     * (not configured down nor link down nor in spanning tree blocking state)
+     */
+    public boolean portEnabled(int portName);
+
+    /**
+     * @param portNumber
+     * @return Whether a port is enabled per latest port status message
+     * (not configured down nor link down nor in spanning tree blocking state)
+     */
+    public boolean portEnabled(String portName);
+
+    /**
+     * Compute the changes that would be required to replace the old ports
+     * of this switch with the new ports.
+     * @param ports new ports to set
+     * @return the ordered collection of changes "applied" to the old ports
+     * of the switch in order to set them to the new set.
+     * If portName <-> portNumber mappings have
+     * changed, the iteration order ensures that delete events for old
+     * conflicting appear before before events adding new ports
+     */
+    public OrderedCollection<PortChangeEvent>
+            comparePorts(Collection<OFPortDesc> ports);
+
+    /**
+     * Replace the ports of this switch with the given ports.
+     * @param ports new ports to set
+     * @return the ordered collection of changes "applied" to the old ports
+     * of the switch in order to set them to the new set.
+     * If portName <-> portNumber mappings have
+     * changed, the iteration order ensures that delete events for old
+     * conflicting appear before before events adding new ports
+     */
+    public OrderedCollection<PortChangeEvent>
+            setPorts(Collection<OFPortDesc> ports);
+
+//  XXX S The odd use of providing an API call to 'set ports' (above) would
+//  logically suggest that there should be a way to delete or unset the ports.
+//  Right now we forbid this. We should probably not use setPorts too.
+//
+//  /**
+//   * Delete a port for the switch. This is called by the core controller
+//   * code in response to a OFPortStatus message. It should not typically be
+//   * called by other floodlight applications.
+//   *
+//   * @param portNumber
+//   */
+//  public void deletePort(short portNumber);
+//
+//  /**
+//   * Delete a port for the switch. This is called by the core controller
+//   * code in response to a OFPortStatus message. It should not typically be
+//   * called by other floodlight applications.
+//   *
+//   * @param portName
+//   */
+//  public void deletePort(String portName);
+
+
+    //*******************************************
+    //  IOFSwitch object attributes
+    //************************
+
+    /**
+     * Gets attributes of this switch.
+     *
+     * @return attributes of the switch
+     */
+    public Map<Object, Object> getAttributes();
+
+    /**
+     * Checks if a specific switch property exists for this switch.
+     *
+     * @param name name of property
+     * @return value for name
+     */
+    boolean hasAttribute(String name);
+
+    /**
+     * Gets properties for switch specific behavior.
+     *
+     * @param name name of property
+     * @return 'value' for 'name', or null if no entry for 'name' exists
+     */
+    Object getAttribute(String name);
+
+    /**
+     * Sets properties for switch specific behavior.
+     *
+     * @param name  name of property
+     * @param value value for name
+     */
+    void setAttribute(String name, Object value);
+
+    /**
+     * Removes properties for switch specific behavior.
+     *
+     * @param name name of property
+     * @return current value for name or null (if not present)
+     */
+    Object removeAttribute(String name);
+
+    //************************
+    //  Switch statistics
+    //************************
+
+    /**
+     * Delivers the statistics future reply.
+     *
+     * @param reply the reply to deliver
+     */
+    public void deliverStatisticsReply(OFMessage reply);
+
+    /**
+     * Cancels the statistics reply with the given transaction ID.
+     *
+     * @param transactionId the transaction ID
+     */
+    public void cancelStatisticsReply(int transactionId);
+
+    /**
+     * Cancels all statistics replies.
+     */
+    public void cancelAllStatisticsReplies();
+
+    /**
+     * Gets a Future object that can be used to retrieve the asynchronous.
+     * OFStatisticsReply when it is available.
+     *
+     * @param request statistics request
+     * @return Future object wrapping OFStatisticsReply
+     * @throws IOException
+     */
+    public Future<List<OFStatsReply>> getStatistics(OFStatsRequest<?> request)
+            throws IOException;
+
+    //************************
+    //  Switch other utilities
+    //************************
+
+    /**
+     * Clears all flowmods on this switch.
+     */
+    public void clearAllFlowMods();
+
+    /**
+     * Gets the current role of this controller for this IOFSwitch.
+     */
+    public Role getRole();
+
+    /**
+     * Sets this controller's Role for this IOFSwitch to role.
+     *
+     * @param role
+     */
+    public void setRole(Role role);
+
+    /**
+     * Gets the next generation ID.
+     * <p>
+     * Note: relevant for role request messages in OF1.3
+     *
+     * @return next generation ID
+     */
+    public U64 getNextGenerationId();
+
+
+    /**
+     * Set debug counter service for per-switch counters.
+     * Called immediately after instantiation.
+     * @param debugCounters
+     * @throws CounterException
+     */
+    public void setDebugCounterService(IDebugCounterService debugCounter)
+            throws CounterException;
+
+    /**
+     * Start this switch driver's sub handshake. This might be a no-op but
+     * this method must be called at least once for the switch to be become
+     * ready.
+     * This method must only be called from the I/O thread
+     * @throws IOException
+     * @throws SwitchDriverSubHandshakeAlreadyStarted if the sub-handshake has
+     * already been started
+     */
+    public void startDriverHandshake() throws IOException;
+
+    /**
+     * Check if the sub-handshake for this switch driver has been completed.
+     * This method can only be called after startDriverHandshake()
+     *
+     * This methods must only be called from the I/O thread
+     * @return true if the sub-handshake has been completed. False otherwise
+     * @throws SwitchDriverSubHandshakeNotStarted if startDriverHandshake() has
+     * not been called yet.
+     */
+    public boolean isDriverHandshakeComplete();
+
+    /**
+     * Pass the given OFMessage to the driver as part of this driver's
+     * sub-handshake. Must not be called after the handshake has been completed
+     * This methods must only be called from the I/O thread
+     * @param m The message that the driver should process
+     * @throws SwitchDriverSubHandshakeCompleted if isDriverHandshake() returns
+     * false before this method call
+     * @throws SwitchDriverSubHandshakeNotStarted if startDriverHandshake() has
+     * not been called yet.
+     */
+    public void processDriverHandshakeMessage(OFMessage m);
+
+    /**
+     * Set the flow table full flag in the switch.
+     * XXX S Rethink this for multiple tables
+     */
+    public void setTableFull(boolean isFull);
+
+    /**
+     * Save the features reply for this switch.
+     *
+     * @param featuresReply
+     */
+    public void setFeaturesReply(OFFeaturesReply featuresReply);
+
+    /**
+     * Save the portset for this switch.
+     *
+     * @param portDescReply
+     */
+    public void setPortDescReply(OFPortDescStatsReply portDescReply);
+
+    //************************
+    //  Message handling
+    //************************
+    /**
+     * Handle the message coming from the dataplane.
+     *
+     * @param m the actual message
+     */
+    public void handleMessage(OFMessage m);
+
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/IOFSwitchManager.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/IOFSwitchManager.java
new file mode 100644
index 0000000..b3b8ed3
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/IOFSwitchManager.java
@@ -0,0 +1,33 @@
+package net.onrc.onos.of.ctl;
+
+import org.projectfloodlight.openflow.protocol.OFVersion;
+
+import net.onrc.onos.of.ctl.registry.IControllerRegistry;
+
+/**
+ * Interface to passed to controller class in order to allow
+ * it to spawn the appropriate type of switch and furthermore
+ * specify a registry object (ie. ZooKeeper).
+ *
+ */
+public interface IOFSwitchManager {
+
+    /**
+     * Given a description string for a switch spawn the
+     * concrete representation of that switch.
+     *
+     * @param mfr manufacturer description
+     * @param hwDesc hardware description
+     * @param swDesc software description
+     * @param ofv openflow version
+     * @return A switch of type IOFSwitch.
+     */
+    public IOFSwitch getSwitchImpl(String mfr, String hwDesc, String swDesc, OFVersion ofv);
+
+    /**
+     * Returns the mastership registry used during controller-switch role election.
+     * @return the registry
+     */
+    public IControllerRegistry getRegistry();
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/Role.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/Role.java
new file mode 100644
index 0000000..d892161
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/Role.java
@@ -0,0 +1,36 @@
+package net.onrc.onos.of.ctl;
+
+import org.projectfloodlight.openflow.protocol.OFControllerRole;
+
+/**
+ * The role of the controller as it pertains to a particular switch.
+ * Note that this definition of the role enum is different from the
+ * OF1.3 definition. It is maintained here to be backward compatible to
+ * earlier versions of the controller code. This enum is translated
+ * to the OF1.3 enum, before role messages are sent to the switch.
+ * See sendRoleRequestMessage method in OFSwitchImpl
+ */
+public enum Role {
+    EQUAL(OFControllerRole.ROLE_EQUAL),
+    MASTER(OFControllerRole.ROLE_MASTER),
+    SLAVE(OFControllerRole.ROLE_SLAVE);
+
+    private Role(OFControllerRole nxRole) {
+        nxRole.ordinal();
+    }
+    /*
+    private static Map<Integer,Role> nxRoleToEnum
+            = new HashMap<Integer,Role>();
+    static {
+        for(Role r: Role.values())
+            nxRoleToEnum.put(r.toNxRole(), r);
+    }
+    public int toNxRole() {
+        return nxRole;
+    }
+    // Return the enum representing the given nxRole or null if no
+    // such role exists
+    public static Role fromNxRole(int nxRole) {
+        return nxRoleToEnum.get(nxRole);
+    }*/
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageCategory.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageCategory.java
new file mode 100644
index 0000000..37ac321
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageCategory.java
@@ -0,0 +1,36 @@
+/**
+ *    Copyright 2012, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation used to set the category for log messages for a class.
+ *
+ */
+@Target({ ElementType.TYPE, ElementType.METHOD })
+public @interface LogMessageCategory {
+
+    /**
+     * The category for the log messages for this class.
+     *
+     * @return
+     */
+    String value() default "Core";
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageDoc.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageDoc.java
new file mode 100644
index 0000000..313e074
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageDoc.java
@@ -0,0 +1,78 @@
+/**
+ *    Copyright 2012, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation used to document log messages.  This can be used to generate
+ * documentation on syslog output.
+ *
+ */
+@Target({ ElementType.TYPE, ElementType.METHOD })
+public @interface LogMessageDoc {
+    public static final String NO_ACTION = "No action is required.";
+    public static final String UNKNOWN_ERROR = "An unknown error occured";
+    public static final String GENERIC_ACTION =
+            "Examine the returned error or exception and take " +
+                    "appropriate action.";
+    public static final String CHECK_SWITCH =
+            "Check the health of the indicated switch.  " +
+                    "Test and troubleshoot IP connectivity.";
+    public static final String CHECK_CONTROLLER =
+            "Verify controller system health, CPU usage, and memory.  " +
+                    "Rebooting the controller node may help if the controller " +
+                    "node is in a distressed state.";
+    public static final String REPORT_CONTROLLER_BUG =
+            "This is likely a defect in the controller.  Please report this " +
+                    "issue.  Restarting the controller or switch may help to " +
+                    "alleviate.";
+    public static final String REPORT_SWITCH_BUG =
+            "This is likely a defect in the switch.  Please report this " +
+                    "issue.  Restarting the controller or switch may help to " +
+                    "alleviate.";
+
+    /**
+     * The log level for the log message.
+     *
+     * @return the log level as a tring
+     */
+    String level() default "INFO";
+
+    /**
+     * The message that will be printed.
+     *
+     * @return the message
+     */
+    String message() default UNKNOWN_ERROR;
+
+    /**
+     * An explanation of the meaning of the log message.
+     *
+     * @return the explanation
+     */
+    String explanation() default UNKNOWN_ERROR;
+
+    /**
+     * The recommendated action associated with the log message.
+     *
+     * @return the recommendation
+     */
+    String recommendation() default NO_ACTION;
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageDocs.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageDocs.java
new file mode 100644
index 0000000..74d4405
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/annotations/LogMessageDocs.java
@@ -0,0 +1,37 @@
+/**
+ *    Copyright 2012, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation used to document log messages.  This can be used to generate
+ * documentation on syslog output.  This version allows multiple log messages
+ * to be documentated on an interface.
+ *
+ */
+@Target({ ElementType.TYPE, ElementType.METHOD })
+public @interface LogMessageDocs {
+    /**
+     * A list of {@link LogMessageDoc} elements.
+     *
+     * @return the list of log message doc
+     */
+    LogMessageDoc[] value();
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/DebugCounter.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/DebugCounter.java
new file mode 100644
index 0000000..6ea380c
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/DebugCounter.java
@@ -0,0 +1,726 @@
+package net.onrc.onos.of.ctl.debugcounter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+import com.google.common.collect.Sets;
+
+
+
+/**
+ * This class implements a central store for all counters used for debugging the
+ * system. For counters based on traffic-type, see ICounterStoreService.
+ *
+ */
+public class DebugCounter implements IDebugCounterService {
+    protected static final Logger log = LoggerFactory.getLogger(DebugCounter.class);
+
+    /**
+     * registered counters need a counter id.
+     */
+    protected AtomicInteger counterIdCounter = new AtomicInteger();
+
+    /**
+     * The counter value.
+     */
+    protected static class MutableLong {
+        long value = 0;
+        public void increment() { value += 1; }
+        public void increment(long incr) { value += incr; }
+        public long get() { return value; }
+        public void set(long val) { value = val; }
+      }
+
+    /**
+     * protected class to store counter information.
+     */
+    public static class CounterInfo {
+        String moduleCounterHierarchy;
+        String counterDesc;
+        CounterType ctype;
+        String moduleName;
+        String counterHierarchy;
+        int counterId;
+        boolean enabled;
+        String[] metaData;
+
+        public CounterInfo(int counterId, boolean enabled,
+                           String moduleName, String counterHierarchy,
+                           String desc, CounterType ctype, String... metaData) {
+            this.moduleCounterHierarchy = moduleName + "/" + counterHierarchy;
+            this.moduleName = moduleName;
+            this.counterHierarchy = counterHierarchy;
+            this.counterDesc = desc;
+            this.ctype = ctype;
+            this.counterId = counterId;
+            this.enabled = enabled;
+            this.metaData = metaData;
+        }
+
+        public String getModuleCounterHierarchy() { return moduleCounterHierarchy; }
+        public String getCounterDesc() { return counterDesc; }
+        public CounterType getCtype() { return ctype; }
+        public String getModuleName() { return moduleName; }
+        public String getCounterHierarchy() { return counterHierarchy; }
+        public int getCounterId() { return counterId; }
+        public boolean isEnabled() { return enabled; }
+        public String[] getMetaData() { return this.metaData.clone(); }
+    }
+
+    //******************
+    //   Global stores
+    //******************
+
+    /**
+     * Counter info for a debug counter.
+     */
+    public static class DebugCounterInfo {
+        CounterInfo cinfo;
+        AtomicLong cvalue;
+
+        public DebugCounterInfo(CounterInfo cinfo) {
+            this.cinfo = cinfo;
+            this.cvalue = new AtomicLong();
+        }
+        public CounterInfo getCounterInfo() {
+            return cinfo;
+        }
+        public Long getCounterValue() {
+            return cvalue.get();
+        }
+    }
+
+    /**
+     * Global debug-counter storage across all threads. These are
+     * updated from the local per thread counters by the flush counters method.
+     */
+    private static final DebugCounterInfo[] ALLCOUNTERS =
+                            new DebugCounterInfo[MAX_COUNTERS];
+
+
+    /**
+     * per module counters, indexed by the module name and storing three levels
+     * of Counter information in the form of CounterIndexStore.
+     */
+    protected ConcurrentHashMap<String, ConcurrentHashMap<String, CounterIndexStore>>
+        moduleCounters = new ConcurrentHashMap<String,
+                                                ConcurrentHashMap<String,
+                                                                   CounterIndexStore>>();
+
+    protected static class CounterIndexStore {
+        int index;
+        Map<String, CounterIndexStore> nextLevel;
+
+        public CounterIndexStore(int index, Map<String, CounterIndexStore> cis) {
+            this.index = index;
+            this.nextLevel = cis;
+        }
+    }
+
+    /**
+     * fast global cache for counter ids that are currently active.
+     */
+    protected Set<Integer> currentCounters = Collections.newSetFromMap(
+                                         new ConcurrentHashMap<Integer, Boolean>());
+
+    //******************
+    // Thread local stores
+    //******************
+
+    /**
+     * Thread local storage of counter info.
+     */
+    protected static class LocalCounterInfo {
+        boolean enabled;
+        MutableLong cvalue;
+
+        public LocalCounterInfo(boolean enabled) {
+            this.enabled = enabled;
+            this.cvalue = new MutableLong();
+        }
+    }
+
+    /**
+     * Thread local debug counters used for maintaining counters local to a thread.
+     */
+    protected final ThreadLocal<LocalCounterInfo[]> threadlocalCounters =
+            new ThreadLocal<LocalCounterInfo[]>() {
+        @Override
+        protected LocalCounterInfo[] initialValue() {
+            return new LocalCounterInfo[MAX_COUNTERS];
+        }
+    };
+
+    /**
+     * Thread local cache for counter ids that are currently active.
+     */
+    protected final ThreadLocal<Set<Integer>> threadlocalCurrentCounters =
+            new ThreadLocal<Set<Integer>>() {
+        @Override
+        protected Set<Integer> initialValue() {
+            return new HashSet<Integer>();
+        }
+    };
+
+    //*******************************
+    //   IDebugCounter
+    //*******************************
+
+    protected class CounterImpl implements IDebugCounter {
+        private final int counterId;
+
+        public CounterImpl(int counterId) {
+            this.counterId = counterId;
+        }
+
+        @Override
+        public void updateCounterWithFlush() {
+            if (!validCounterId()) {
+                return;
+            }
+            updateCounter(counterId, 1, true);
+        }
+
+        @Override
+        public void updateCounterNoFlush() {
+            if (!validCounterId()) {
+                return;
+            }
+            updateCounter(counterId, 1, false);
+        }
+
+        @Override
+        public void updateCounterWithFlush(int incr) {
+            if (!validCounterId()) {
+                return;
+            }
+            updateCounter(counterId, incr, true);
+        }
+
+        @Override
+        public void updateCounterNoFlush(int incr) {
+            if (!validCounterId()) {
+                return;
+            }
+            updateCounter(counterId, incr, false);
+        }
+
+        @Override
+        public long getCounterValue() {
+            if (!validCounterId()) {
+                return -1;
+            }
+            return ALLCOUNTERS[counterId].cvalue.get();
+        }
+
+        /**
+         * Checks if this is a valid counter.
+         * @return true if the counter id is valid
+         */
+        private boolean validCounterId() {
+            if (counterId < 0 || counterId >= MAX_COUNTERS) {
+                log.error("Invalid counterId invoked");
+                return false;
+            }
+            return true;
+        }
+
+    }
+
+   //*******************************
+   //   IDebugCounterService
+   //*******************************
+
+   @Override
+   public IDebugCounter registerCounter(String moduleName, String counterHierarchy,
+                           String counterDescription, CounterType counterType,
+                           String... metaData)
+               throws CounterException {
+       // check if counter already exists
+       if (!moduleCounters.containsKey(moduleName)) {
+           moduleCounters.putIfAbsent(moduleName,
+                new ConcurrentHashMap<String, CounterIndexStore>());
+       }
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (rci.allLevelsFound) {
+           // counter exists
+           log.info("Counter exists for {}/{} -- resetting counters", moduleName,
+                    counterHierarchy);
+           resetCounterHierarchy(moduleName, counterHierarchy);
+           return new CounterImpl(rci.ctrIds[rci.foundUptoLevel - 1]);
+       }
+       // check for validity of counter
+       if (rci.levels.length > MAX_HIERARCHY) {
+           String err = "Registry of counterHierarchy " + counterHierarchy +
+                   " exceeds max hierachy " + MAX_HIERARCHY + ".. aborting";
+           throw new MaxHierarchyRegistered(err);
+       }
+       if (rci.foundUptoLevel < rci.levels.length - 1) {
+           StringBuilder sb = new StringBuilder();
+           for (int i = 0; i <= rci.foundUptoLevel; i++) {
+               sb.append(rci.levels[i]);
+           }
+           String needToRegister = sb.toString();
+           String err = "Attempting to register hierarchical counterHierarchy " +
+                   counterHierarchy + " but parts of hierarchy missing. " +
+                   "Please register " +  needToRegister + " first";
+           throw new MissingHierarchicalLevel(err);
+       }
+
+       // get a new counter id
+       int counterId = counterIdCounter.getAndIncrement();
+       if (counterId >= MAX_COUNTERS) {
+           throw new MaxCountersRegistered("max counters reached");
+       }
+       // create storage for counter
+       boolean enabled = (counterType == CounterType.ALWAYS_COUNT) ? true : false;
+       CounterInfo ci = new CounterInfo(counterId, enabled, moduleName,
+                                        counterHierarchy, counterDescription,
+                                        counterType, metaData);
+       ALLCOUNTERS[counterId] = new DebugCounterInfo(ci);
+
+       // account for the new counter in the module counter hierarchy
+       addToModuleCounterHierarchy(moduleName, counterId, rci);
+
+       // finally add to active counters
+       if (enabled) {
+           currentCounters.add(counterId);
+       }
+       return new CounterImpl(counterId);
+   }
+
+   private void updateCounter(int counterId, int incr, boolean flushNow) {
+       if (counterId < 0 || counterId >= MAX_COUNTERS) {
+        return;
+    }
+
+       LocalCounterInfo[] thiscounters =  this.threadlocalCounters.get();
+       if (thiscounters[counterId] == null) {
+           // seeing this counter for the first time in this thread - create local
+           // store by consulting global store
+           DebugCounterInfo dc = ALLCOUNTERS[counterId];
+           if (dc != null) {
+               thiscounters[counterId] = new LocalCounterInfo(dc.cinfo.enabled);
+               if (dc.cinfo.enabled) {
+                   Set<Integer> thisset = this.threadlocalCurrentCounters.get();
+                   thisset.add(counterId);
+               }
+           } else {
+               log.error("updateCounter seen locally for counter {} but no global"
+                          + "storage exists for it yet .. not updating", counterId);
+               return;
+           }
+       }
+
+       // update local store if enabled locally for updating
+       LocalCounterInfo lc = thiscounters[counterId];
+       if (lc.enabled) {
+           lc.cvalue.increment(incr);
+           if (flushNow) {
+               DebugCounterInfo dc = ALLCOUNTERS[counterId];
+               if (dc.cinfo.enabled) {
+                   // globally enabled - flush now
+                   dc.cvalue.addAndGet(lc.cvalue.get());
+                   lc.cvalue.set(0);
+               } else {
+                   // global counter is disabled - don't flush, disable locally
+                   lc.enabled = false;
+                   Set<Integer> thisset = this.threadlocalCurrentCounters.get();
+                   thisset.remove(counterId);
+               }
+           }
+       }
+   }
+
+   @Override
+   public void flushCounters() {
+       LocalCounterInfo[] thiscounters =  this.threadlocalCounters.get();
+       Set<Integer> thisset = this.threadlocalCurrentCounters.get();
+       ArrayList<Integer> temp = new ArrayList<Integer>();
+
+       for (int counterId : thisset) {
+           LocalCounterInfo lc = thiscounters[counterId];
+           if (lc.cvalue.get() > 0) {
+               DebugCounterInfo dc = ALLCOUNTERS[counterId];
+               if (dc.cinfo.enabled) {
+                   // globally enabled - flush now
+                   dc.cvalue.addAndGet(lc.cvalue.get());
+                   lc.cvalue.set(0);
+               } else {
+                   // global counter is disabled - don't flush, disable locally
+                   lc.enabled = false;
+                   temp.add(counterId);
+               }
+           }
+       }
+       for (int cId : temp) {
+           thisset.remove(cId);
+       }
+
+       // At this point it is possible that the thread-local set does not
+       // include a counter that has been enabled and is present in the global set.
+       // We need to sync thread-local currently enabled set of counterIds with
+       // the global set.
+       Sets.SetView<Integer> sv = Sets.difference(currentCounters, thisset);
+       for (int counterId : sv) {
+           if (thiscounters[counterId] != null) {
+               thiscounters[counterId].enabled = true;
+               thisset.add(counterId);
+           }
+       }
+   }
+
+   @Override
+   public void resetCounterHierarchy(String moduleName, String counterHierarchy) {
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (!rci.allLevelsFound) {
+           String missing = rci.levels[rci.foundUptoLevel];
+           log.error("Cannot reset counter hierarchy - missing counter {}", missing);
+           return;
+       }
+       // reset at this level
+       ALLCOUNTERS[rci.ctrIds[rci.foundUptoLevel - 1]].cvalue.set(0);
+       // reset all levels below
+       ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+       for (int index : resetIds) {
+           ALLCOUNTERS[index].cvalue.set(0);
+       }
+   }
+
+   @Override
+   public void resetAllCounters() {
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+       for (String moduleName : moduleCounters.keySet()) {
+           ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+           for (int index : resetIds) {
+               ALLCOUNTERS[index].cvalue.set(0);
+           }
+       }
+   }
+
+   @Override
+   public void resetAllModuleCounters(String moduleName) {
+       Map<String, CounterIndexStore> target = moduleCounters.get(moduleName);
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+
+       if (target != null) {
+           ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+           for (int index : resetIds) {
+               ALLCOUNTERS[index].cvalue.set(0);
+           }
+       } else {
+           if (log.isDebugEnabled()) {
+            log.debug("No module found with name {}", moduleName);
+        }
+       }
+   }
+
+   @Override
+   public void enableCtrOnDemand(String moduleName, String counterHierarchy) {
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (!rci.allLevelsFound) {
+           String missing = rci.levels[rci.foundUptoLevel];
+           log.error("Cannot enable counter - counter not found {}", missing);
+           return;
+       }
+       // enable specific counter
+       DebugCounterInfo dc = ALLCOUNTERS[rci.ctrIds[rci.foundUptoLevel - 1]];
+       dc.cinfo.enabled = true;
+       currentCounters.add(dc.cinfo.counterId);
+   }
+
+   @Override
+   public void disableCtrOnDemand(String moduleName, String counterHierarchy) {
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (!rci.allLevelsFound) {
+           String missing = rci.levels[rci.foundUptoLevel];
+           log.error("Cannot disable counter - counter not found {}", missing);
+           return;
+       }
+       // disable specific counter
+       DebugCounterInfo dc = ALLCOUNTERS[rci.ctrIds[rci.foundUptoLevel - 1]];
+       if (dc.cinfo.ctype == CounterType.COUNT_ON_DEMAND) {
+           dc.cinfo.enabled = false;
+           dc.cvalue.set(0);
+           currentCounters.remove(dc.cinfo.counterId);
+       }
+   }
+
+   @Override
+   public List<DebugCounterInfo> getCounterHierarchy(String moduleName,
+                                                     String counterHierarchy) {
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (!rci.allLevelsFound) {
+           String missing = rci.levels[rci.foundUptoLevel];
+           log.error("Cannot fetch counter - counter not found {}", missing);
+           return Collections.emptyList();
+       }
+       ArrayList<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>();
+       // get counter and all below it
+       DebugCounterInfo dc = ALLCOUNTERS[rci.ctrIds[rci.foundUptoLevel - 1]];
+       dcilist.add(dc);
+       ArrayList<Integer> belowIds = getHierarchyBelow(moduleName, rci);
+       for (int index : belowIds) {
+           dcilist.add(ALLCOUNTERS[index]);
+       }
+       return dcilist;
+   }
+
+   @Override
+   public List<DebugCounterInfo> getAllCounterValues() {
+       List<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>();
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+
+       for (String moduleName : moduleCounters.keySet()) {
+           ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+           for (int index : resetIds) {
+               dcilist.add(ALLCOUNTERS[index]);
+           }
+       }
+       return dcilist;
+   }
+
+   @Override
+   public List<DebugCounterInfo> getModuleCounterValues(String moduleName) {
+       List<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>();
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+
+       if (moduleCounters.containsKey(moduleName)) {
+           ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+           for (int index : resetIds) {
+               dcilist.add(ALLCOUNTERS[index]);
+           }
+       }
+       return dcilist;
+   }
+
+   @Override
+   public boolean containsModuleCounterHierarchy(String moduleName,
+                                                 String counterHierarchy) {
+       if (!moduleCounters.containsKey(moduleName)) {
+        return false;
+    }
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       return rci.allLevelsFound;
+   }
+
+   @Override
+   public boolean containsModuleName(String moduleName) {
+       return  (moduleCounters.containsKey(moduleName)) ? true : false;
+   }
+
+   @Override
+   public List<String> getModuleList() {
+       List<String> retval = new ArrayList<String>();
+       retval.addAll(moduleCounters.keySet());
+       return retval;
+   }
+
+   @Override
+   public List<String> getModuleCounterList(String moduleName) {
+       if (!moduleCounters.containsKey(moduleName)) {
+        return Collections.emptyList();
+    }
+
+       List<String> retval = new ArrayList<String>();
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+
+       ArrayList<Integer> cids = getHierarchyBelow(moduleName, rci);
+       for (int index : cids) {
+           retval.add(ALLCOUNTERS[index].cinfo.counterHierarchy);
+       }
+       return retval;
+   }
+
+   //*******************************
+   //   Internal Methods
+   //*******************************
+
+   protected class RetCtrInfo {
+       boolean allLevelsFound; // counter indices found all the way down the hierarchy
+       boolean hierarchical; // true if counterHierarchy is hierarchical
+       int foundUptoLevel;
+       int[]  ctrIds;
+       String[] levels;
+
+       public RetCtrInfo() {
+           ctrIds = new int[MAX_HIERARCHY];
+           for (int i = 0; i < MAX_HIERARCHY; i++) {
+               ctrIds[i] = -1;
+           }
+       }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + getOuterType().hashCode();
+        result = prime * result + (allLevelsFound ? 1231 : 1237);
+        result = prime * result + Arrays.hashCode(ctrIds);
+        result = prime * result + foundUptoLevel;
+        result = prime * result + (hierarchical ? 1231 : 1237);
+        result = prime * result + Arrays.hashCode(levels);
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object oth) {
+        if (!(oth instanceof RetCtrInfo)) {
+         return false;
+     }
+        RetCtrInfo other = (RetCtrInfo) oth;
+        if (other.allLevelsFound != this.allLevelsFound) {
+         return false;
+     }
+        if (other.hierarchical != this.hierarchical) {
+         return false;
+     }
+        if (other.foundUptoLevel != this.foundUptoLevel) {
+         return false;
+     }
+        if (!Arrays.equals(other.ctrIds, this.ctrIds)) {
+         return false;
+     }
+        if (!Arrays.equals(other.levels, this.levels)) {
+         return false;
+     }
+        return true;
+    }
+
+    private DebugCounter getOuterType() {
+        return DebugCounter.this;
+    }
+
+
+
+   }
+
+   protected RetCtrInfo getCounterId(String moduleName, String counterHierarchy) {
+       RetCtrInfo rci = new RetCtrInfo();
+       Map<String, CounterIndexStore> templevel = moduleCounters.get(moduleName);
+       rci.levels = counterHierarchy.split("/");
+       if (rci.levels.length > 1) {
+        rci.hierarchical = true;
+    }
+       if (templevel == null) {
+           log.error("moduleName {} does not exist in debugCounters", moduleName);
+           return rci;
+       }
+
+       /*
+       if (rci.levels.length > MAX_HIERARCHY) {
+           // chop off all array elems greater that MAX_HIERARCHY
+           String[] temp = new String[MAX_HIERARCHY];
+           System.arraycopy(rci.levels, 0, temp, 0, MAX_HIERARCHY);
+           rci.levels = temp;
+       }
+       */
+       for (int i = 0; i < rci.levels.length; i++) {
+           if (templevel != null) {
+               CounterIndexStore cis = templevel.get(rci.levels[i]);
+               if (cis == null) {
+                   // could not find counterHierarchy part at this level
+                   break;
+               } else {
+                   rci.ctrIds[i] = cis.index;
+                   templevel = cis.nextLevel;
+                   rci.foundUptoLevel++;
+                   if (i == rci.levels.length - 1) {
+                       rci.allLevelsFound = true;
+                   }
+               }
+           } else {
+               // there are no more levels, which means that some part of the
+               // counterHierarchy has no corresponding map
+               break;
+           }
+       }
+       return rci;
+   }
+
+   protected void addToModuleCounterHierarchy(String moduleName, int counterId,
+                                            RetCtrInfo rci) {
+       Map<String, CounterIndexStore> target = moduleCounters.get(moduleName);
+       if (target == null) {
+        return;
+    }
+       CounterIndexStore cis = null;
+
+       for (int i = 0; i < rci.foundUptoLevel; i++) {
+           cis = target.get(rci.levels[i]);
+           target = cis.nextLevel;
+       }
+       if (cis != null) {
+           if (cis.nextLevel == null) {
+            cis.nextLevel = new ConcurrentHashMap<String, CounterIndexStore>();
+        }
+           cis.nextLevel.put(rci.levels[rci.foundUptoLevel],
+                             new CounterIndexStore(counterId, null));
+       } else {
+           target.put(rci.levels[rci.foundUptoLevel],
+                      new CounterIndexStore(counterId, null));
+       }
+   }
+
+   // given a partial hierarchical counter, return the rest of the hierarchy
+   protected ArrayList<Integer> getHierarchyBelow(String moduleName, RetCtrInfo rci) {
+       Map<String, CounterIndexStore> target = moduleCounters.get(moduleName);
+       CounterIndexStore cis = null;
+       ArrayList<Integer> retval = new ArrayList<Integer>();
+       if (target == null) {
+        return retval;
+    }
+
+       // get to the level given
+       for (int i = 0; i < rci.foundUptoLevel; i++) {
+           cis = target.get(rci.levels[i]);
+           target = cis.nextLevel;
+       }
+
+       if (target == null || rci.foundUptoLevel == MAX_HIERARCHY) {
+           // no more levels
+           return retval;
+       } else {
+           // recursively get all ids
+           getIdsAtLevel(target, retval, rci.foundUptoLevel + 1);
+       }
+
+       return retval;
+   }
+
+   protected void getIdsAtLevel(Map<String, CounterIndexStore> hcy,
+                                ArrayList<Integer> retval, int level) {
+       if (level > MAX_HIERARCHY) {
+        return;
+    }
+       if (hcy == null || retval == null) {
+        return;
+    }
+
+       // Can return the counter names as well but for now ids are enough.
+       for (CounterIndexStore cistemp : hcy.values()) {
+           retval.add(cistemp.index); // value at this level
+           if (cistemp.nextLevel != null) {
+               getIdsAtLevel(cistemp.nextLevel, retval, level + 1);
+           }
+       }
+   }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/IDebugCounter.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/IDebugCounter.java
new file mode 100644
index 0000000..e157cf2
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/IDebugCounter.java
@@ -0,0 +1,38 @@
+package net.onrc.onos.of.ctl.debugcounter;
+
+public interface IDebugCounter {
+    /**
+     * Increments the counter by 1 thread-locally, and immediately flushes to
+     * the global counter storage. This method should be used for counters that
+     * are updated outside the OF message processing pipeline.
+     */
+    void updateCounterWithFlush();
+
+    /**
+     * Increments the counter by 1 thread-locally. Flushing to the global
+     * counter storage is delayed (happens with flushCounters() in IDebugCounterService),
+     * resulting in higher performance. This method should be used for counters
+     * updated in the OF message processing pipeline.
+     */
+    void updateCounterNoFlush();
+
+    /**
+     * Increments the counter thread-locally by the 'incr' specified, and immediately
+     * flushes to the global counter storage. This method should be used for counters
+     * that are updated outside the OF message processing pipeline.
+     */
+    void updateCounterWithFlush(int incr);
+
+    /**
+     * Increments the counter thread-locally by the 'incr' specified. Flushing to the global
+     * counter storage is delayed (happens with flushCounters() in IDebugCounterService),
+     * resulting in higher performance. This method should be used for counters
+     * updated in the OF message processing pipeline.
+     */
+    void updateCounterNoFlush(int incr);
+
+    /**
+     * Retrieve the value of the counter from the global counter store.
+     */
+    long getCounterValue();
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/IDebugCounterService.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/IDebugCounterService.java
new file mode 100644
index 0000000..622f647
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/IDebugCounterService.java
@@ -0,0 +1,260 @@
+package net.onrc.onos.of.ctl.debugcounter;
+
+
+
+import java.util.List;
+
+import net.onrc.onos.of.ctl.debugcounter.DebugCounter.DebugCounterInfo;
+
+public interface IDebugCounterService {
+
+    /**
+     * Different counter types. Counters that are meant to be counted-on-demand
+     * need to be separately enabled/disabled.
+     */
+    public enum CounterType {
+        ALWAYS_COUNT,
+        COUNT_ON_DEMAND
+    }
+
+    /**
+     * Debug Counter Qualifiers.
+     */
+    public static final String CTR_MDATA_WARN = "warn";
+    public static final String CTR_MDATA_ERROR = "error";
+    public static final String CTR_MDATA_DROP = "drop";
+
+    /**
+     *  A limit on the maximum number of counters that can be created.
+     */
+    public static final int MAX_COUNTERS = 5000;
+
+    /**
+     * Exception thrown when MAX_COUNTERS have been registered.
+     */
+    public class MaxCountersRegistered extends CounterException {
+        private static final long serialVersionUID = 3173747663719376745L;
+        String errormsg;
+        public MaxCountersRegistered(String errormsg) {
+            this.errormsg = errormsg;
+        }
+        @Override
+        public String getMessage() {
+            return this.errormsg;
+        }
+    }
+    /**
+     * Exception thrown when MAX_HIERARCHY has been reached.
+     */
+    public class MaxHierarchyRegistered extends CounterException {
+        private static final long serialVersionUID = 967431358683523871L;
+        private String errormsg;
+        public MaxHierarchyRegistered(String errormsg) {
+            this.errormsg = errormsg;
+        }
+        @Override
+        public String getMessage() {
+            return this.errormsg;
+        }
+    }
+    /**
+     * Exception thrown when attempting to register a hierarchical counter
+     * where higher levels of the hierarchy have not been pre-registered.
+     */
+    public class MissingHierarchicalLevel extends CounterException {
+        private static final long serialVersionUID = 517315311533995739L;
+        private String errormsg;
+        public MissingHierarchicalLevel(String errormsg) {
+            this.errormsg = errormsg;
+        }
+        @Override
+        public String getMessage() {
+            return this.errormsg;
+        }
+    }
+
+    public class CounterException extends Exception {
+        private static final long serialVersionUID = 2219781500857866035L;
+    }
+
+    /**
+     *  maximum levels of hierarchy.
+     *  Example of moduleName/counterHierarchy:
+     *           switch/00:00:00:00:01:02:03:04/pktin/drops where
+     *           moduleName ==> "switch"  and
+     *           counterHierarchy of 3 ==> "00:00:00:00:01:02:03:04/pktin/drops"
+     */
+    public static final int MAX_HIERARCHY = 3;
+
+    /**
+     * All modules that wish to have the DebugCounterService count for them, must
+     * register their counters by making this call (typically from that module's
+     * 'startUp' method). The counter can then be updated, displayed, reset etc.
+     * using the registered moduleName and counterHierarchy.
+     *
+     * @param moduleName           the name of the module which is registering the
+     *                             counter eg. linkdiscovery or controller or switch
+     * @param counterHierarchy     the hierarchical counter name specifying all
+     *                             the hierarchical levels that come above it.
+     *                             For example: to register a drop counter for
+     *                             packet-ins from a switch, the counterHierarchy
+     *                             can be "00:00:00:00:01:02:03:04/pktin/drops"
+     *                             It is necessary that counters in hierarchical levels
+     *                             above have already been pre-registered - in this
+     *                             example: "00:00:00:00:01:02:03:04/pktin" and
+     *                             "00:00:00:00:01:02:03:04"
+     * @param counterDescription   a descriptive string that gives more information
+     *                             of what the counter is measuring. For example,
+     *                             "Measures the number of incoming packets seen by
+     *                             this module".
+     * @param counterType          One of CounterType. On-demand counter types
+     *                             need to be explicitly enabled/disabled using other
+     *                             methods in this API -- i.e. registering them is
+     *                             not enough to start counting.
+     * @param metaData             variable arguments that qualify a counter
+     *                             eg. warn, error etc.
+     * @return                     IDebugCounter with update methods that can be
+     *                             used to update a counter.
+     * @throws MaxCountersRegistered
+     * @throws MaxHierarchyRegistered
+     * @throws MissingHierarchicalLevel
+     */
+    public IDebugCounter registerCounter(String moduleName, String counterHierarchy,
+                             String counterDescription, CounterType counterType,
+                             String... metaData)
+                throws CounterException;
+
+    /**
+     * Flush all thread-local counter values (from the current thread)
+     * to the global counter store. This method is not intended for use by any
+     * module. It's typical usage is from floodlight core and it is meant
+     * to flush those counters that are updated in the packet-processing pipeline,
+     * typically with the 'updateCounterNoFlush" methods in IDebugCounter.
+     */
+    public void flushCounters();
+
+    /**
+     * Resets the value of counters in the hierarchy to zero. Note that the reset
+     * applies to the level of counter hierarchy specified AND ALL LEVELS BELOW it
+     * in the hierarchy.
+     * For example: If a hierarchy exists like "00:00:00:00:01:02:03:04/pktin/drops"
+     *              specifying a reset hierarchy: "00:00:00:00:01:02:03:04"
+     *              will reset all counters for the switch dpid specified;
+     *              while specifying a reset hierarchy: ""00:00:00:00:01:02:03:04/pktin"
+     *              will reset the pktin counter and all levels below it (like drops)
+     *              for the switch dpid specified.
+     */
+    void resetCounterHierarchy(String moduleName, String counterHierarchy);
+
+    /**
+     * Resets the values of all counters in the system.
+     */
+    public void resetAllCounters();
+
+    /**
+     * Resets the values of all counters belonging
+     * to a module with the given 'moduleName'.
+     */
+    public void resetAllModuleCounters(String moduleName);
+
+    /**
+     * This method applies only to CounterType.COUNT_ON_DEMAND. It is used to
+     * enable counting on the counter. Note that this step is necessary to start
+     * counting for these counter types - merely registering the counter is not
+     * enough (as is the case for CounterType.ALWAYS_COUNT). Newly
+     * enabled counters start from an initial value of zero.
+     *
+     * Enabling a counter in a counterHierarchy enables only THAT counter. It
+     * does not enable any other part of the counterHierarchy. For example, if
+     * a hierarchy exists like "00:00:00:00:01:02:03:04/pktin/drops", where the
+     * 'pktin' and 'drops' counters are CounterType.COUNT_ON_DEMAND, then enabling
+     * the 'pktin' counter by specifying the counterHierarchy as
+     * "00:00:00:00:01:02:03:04/pktin" does NOT enable the 'drops' counter.
+     */
+    public void enableCtrOnDemand(String moduleName, String counterHierarchy);
+
+    /**
+     * This method applies only to CounterType.COUNT_ON_DEMAND. It is used to
+     * enable counting on the counter. Note that disabling a counter results in a loss
+     * of the counter value. When re-enabled the counter will restart from zero.
+     *
+     * Disabling a counter in a counterHierarchy disables only THAT counter. It
+     * does not disable any other part of the counterHierarchy. For example, if
+     * a hierarchy exists like "00:00:00:00:01:02:03:04/pktin/drops", where the
+     * 'pktin' and 'drops' counters are CounterType.COUNT_ON_DEMAND, then disabling
+     * the 'pktin' counter by specifying the counterHierarchy as
+     * "00:00:00:00:01:02:03:04/pktin" does NOT disable the 'drops' counter.
+     */
+    public void disableCtrOnDemand(String moduleName, String counterHierarchy);
+
+    /**
+     * Get counter value and associated information for the specified counterHierarchy.
+     * Note that information on the level of counter hierarchy specified
+     * AND ALL LEVELS BELOW it in the hierarchy will be returned.
+     *
+     * For example,
+     * if a hierarchy exists like "00:00:00:00:01:02:03:04/pktin/drops", then
+     * specifying a counterHierarchy of "00:00:00:00:01:02:03:04/pktin" in the
+     * get call will return information on the 'pktin' as well as the 'drops'
+     * counters for the switch dpid specified.
+     *
+     * @return A list of DebugCounterInfo or an empty list if the counter
+     *         could not be found
+     */
+    public List<DebugCounterInfo> getCounterHierarchy(String moduleName,
+                                                      String counterHierarchy);
+
+    /**
+     * Get counter values and associated information for all counters in the
+     * system.
+     *
+     * @return the list of values/info or an empty list
+     */
+    public  List<DebugCounterInfo> getAllCounterValues();
+
+    /**
+     * Get counter values and associated information for all counters associated
+     * with a module.
+     *
+     * @param moduleName
+     * @return the list of values/info or an empty list
+     */
+    public  List<DebugCounterInfo> getModuleCounterValues(String moduleName);
+
+    /**
+     * Convenience method to figure out if the the given 'counterHierarchy' corresponds
+     * to a registered counterHierarchy for 'moduleName'. Note that the counter may or
+     * may not be enabled for counting, but if it is registered the method will
+     * return true.
+     *
+     * @param param
+     * @return false if moduleCounterHierarchy is not a registered counter
+     */
+    public boolean containsModuleCounterHierarchy(String moduleName,
+                                                  String counterHierarchy);
+
+    /**
+     * Convenience method to figure out if the the given 'moduleName' corresponds
+     * to a registered moduleName or not. Note that the module may or may not have
+     * a counter enabled for counting, but if it is registered the method will
+     * return true.
+     *
+     * @param param
+     * @return false if moduleName is not a registered counter
+     */
+    public boolean containsModuleName(String moduleName);
+
+    /**
+     * Returns a list of moduleNames registered for debug counters or an empty
+     * list if no counters have been registered in the system.
+     */
+    public List<String> getModuleList();
+
+    /**
+     * Returns a list of all counters registered for a specific moduleName
+     * or a empty list.
+     */
+    public List<String> getModuleCounterList(String moduleName);
+
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/NullDebugCounter.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/NullDebugCounter.java
new file mode 100644
index 0000000..1775c50
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/debugcounter/NullDebugCounter.java
@@ -0,0 +1,116 @@
+package net.onrc.onos.of.ctl.debugcounter;
+
+import java.util.Collections;
+import java.util.List;
+
+import net.onrc.onos.of.ctl.debugcounter.DebugCounter.DebugCounterInfo;
+
+public class NullDebugCounter implements IDebugCounterService {
+
+    @Override
+    public void flushCounters() {
+
+    }
+
+    @Override
+    public void resetAllCounters() {
+
+    }
+
+    @Override
+    public void resetAllModuleCounters(String moduleName) {
+
+    }
+
+
+    @Override
+    public void resetCounterHierarchy(String moduleName, String counterHierarchy) {
+
+    }
+
+    @Override
+    public void enableCtrOnDemand(String moduleName, String counterHierarchy) {
+
+    }
+
+    @Override
+    public void disableCtrOnDemand(String moduleName, String counterHierarchy) {
+
+    }
+
+    @Override
+    public List<DebugCounterInfo> getCounterHierarchy(String moduleName,
+                                                      String counterHierarchy) {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<DebugCounterInfo> getAllCounterValues() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<DebugCounterInfo> getModuleCounterValues(String moduleName) {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public boolean containsModuleCounterHierarchy(String moduleName,
+                                             String counterHierarchy) {
+        return false;
+    }
+
+    @Override
+    public boolean containsModuleName(String moduleName) {
+        return false;
+    }
+
+    @Override
+    public
+            IDebugCounter
+            registerCounter(String moduleName, String counterHierarchy,
+                            String counterDescription,
+                            CounterType counterType, String... metaData)
+                                 throws MaxCountersRegistered {
+        return new NullCounterImpl();
+    }
+
+    @Override
+    public List<String> getModuleList() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<String> getModuleCounterList(String moduleName) {
+        return Collections.emptyList();
+    }
+
+    public static class NullCounterImpl implements IDebugCounter {
+
+        @Override
+        public void updateCounterWithFlush() {
+
+        }
+
+        @Override
+        public void updateCounterNoFlush() {
+
+        }
+
+        @Override
+        public void updateCounterWithFlush(int incr) {
+        }
+
+        @Override
+        public void updateCounterNoFlush(int incr) {
+
+        }
+
+        @Override
+        public long getCounterValue() {
+            return -1;
+        }
+
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/Controller.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/Controller.java
new file mode 100644
index 0000000..05a841d
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/Controller.java
@@ -0,0 +1,863 @@
+/**
+ *    Copyright 2011, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.internal;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+
+import net.onrc.onos.of.ctl.IOFSwitchManager;
+import net.onrc.onos.of.ctl.Role;
+import net.onrc.onos.of.ctl.annotations.LogMessageDoc;
+import net.onrc.onos.of.ctl.annotations.LogMessageDocs;
+import net.onrc.onos.of.ctl.debugcounter.DebugCounter;
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounter;
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounterService;
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounterService.CounterException;
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounterService.CounterType;
+import net.onrc.onos.of.ctl.internal.OFChannelHandler.RoleRecvStatus;
+import net.onrc.onos.of.ctl.registry.IControllerRegistry;
+import net.onrc.onos.of.ctl.registry.RegistryException;
+import net.onrc.onos.of.ctl.registry.IControllerRegistry.ControlChangeCallback;
+import net.onrc.onos.of.ctl.util.Dpid;
+import net.onrc.onos.of.ctl.util.DummySwitchForTesting;
+import net.onrc.onos.of.ctl.util.InstanceId;
+import net.onrc.onos.of.ctl.IOFSwitch;
+import net.onrc.onos.of.ctl.IOFSwitch.PortChangeType;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.jboss.netty.bootstrap.ServerBootstrap;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.group.ChannelGroup;
+import org.jboss.netty.channel.group.DefaultChannelGroup;
+import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFFactories;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.util.HexString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * The main controller class.  Handles all setup and network listeners
+ * - Distributed ownership control of switch through IControllerRegistryService
+ */
+@Component(immediate = true)
+public class Controller {
+
+    protected static final Logger log = LoggerFactory.getLogger(Controller.class);
+    static final String ERROR_DATABASE =
+            "The controller could not communicate with the system database.";
+    protected static final OFFactory FACTORY13 = OFFactories.getFactory(OFVersion.OF_13);
+    protected static final OFFactory FACTORY10 = OFFactories.getFactory(OFVersion.OF_10);
+
+    // connectedSwitches cache contains all connected switch's channelHandlers
+    // including ones where this controller is a master/equal/slave controller
+    // as well as ones that have not been activated yet
+    protected ConcurrentHashMap<Long, OFChannelHandler> connectedSwitches;
+    // These caches contains only those switches that are active
+    protected ConcurrentHashMap<Long, IOFSwitch> activeMasterSwitches;
+    protected ConcurrentHashMap<Long, IOFSwitch> activeEqualSwitches;
+    // lock to synchronize on, when manipulating multiple caches above
+    private Object multiCacheLock;
+
+    // The controllerNodeIPsCache maps Controller IDs to their IP address.
+    // It's only used by handleControllerNodeIPsChanged
+    protected HashMap<String, String> controllerNodeIPsCache;
+
+    // Module dependencies
+
+    protected IControllerRegistry registryService;
+    protected IDebugCounterService debugCounters;
+
+
+    private IOFSwitchManager switchManager;
+
+    // Configuration options
+    protected int openFlowPort = 6633;
+    protected int workerThreads = 0;
+
+    // defined counters
+    private Counters counters;
+
+    // Start time of the controller
+    protected long systemStartTime;
+
+    // Flag to always flush flow table on switch reconnect (HA or otherwise)
+    protected boolean alwaysClearFlowsOnSwAdd = false;
+    private InstanceId instanceId;
+
+    // Perf. related configuration
+    protected static final int SEND_BUFFER_SIZE = 4 * 1024 * 1024;
+    protected static final int BATCH_MAX_SIZE = 100;
+    protected static final boolean ALWAYS_DECODE_ETH = true;
+
+    protected boolean addConnectedSwitch(long dpid, OFChannelHandler h) {
+        if (connectedSwitches.get(dpid) != null) {
+            log.error("Trying to add connectedSwitch but found a previous "
+                    + "value for dpid: {}", dpid);
+            return false;
+        } else {
+            log.error("Added switch {}", dpid);
+            connectedSwitches.put(dpid, h);
+            return true;
+        }
+    }
+
+    private boolean validActivation(long dpid) {
+        if (connectedSwitches.get(dpid) == null) {
+            log.error("Trying to activate switch but is not in "
+                    + "connected switches: dpid {}. Aborting ..",
+                    HexString.toHexString(dpid));
+            return false;
+        }
+        if (activeMasterSwitches.get(dpid) != null ||
+                activeEqualSwitches.get(dpid) != null) {
+            log.error("Trying to activate switch but it is already "
+                    + "activated: dpid {}. Found in activeMaster: {} "
+                    + "Found in activeEqual: {}. Aborting ..", new Object[] {
+                            HexString.toHexString(dpid),
+                            (activeMasterSwitches.get(dpid) == null) ? 'Y' : 'N',
+                            (activeEqualSwitches.get(dpid) == null) ? 'Y' : 'N'});
+            counters.switchWithSameDpidActivated.updateCounterWithFlush();
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Called when a switch is activated, with this controller's role as MASTER.
+     */
+    protected boolean addActivatedMasterSwitch(long dpid, IOFSwitch sw) {
+        synchronized (multiCacheLock) {
+            if (!validActivation(dpid)) {
+                return false;
+            }
+            activeMasterSwitches.put(dpid, sw);
+        }
+        //update counters and events
+        counters.switchActivated.updateCounterWithFlush();
+
+        return true;
+    }
+
+    /**
+     * Called when a switch is activated, with this controller's role as EQUAL.
+     */
+    protected boolean addActivatedEqualSwitch(long dpid, IOFSwitch sw) {
+        synchronized (multiCacheLock) {
+            if (!validActivation(dpid)) {
+                return false;
+            }
+            activeEqualSwitches.put(dpid, sw);
+        }
+        //update counters and events
+        counters.switchActivated.updateCounterWithFlush();
+        return true;
+    }
+
+    /**
+     * Called when this controller's role for a switch transitions from equal
+     * to master. For 1.0 switches, we internally refer to the role 'slave' as
+     * 'equal' - so this transition is equivalent to 'addActivatedMasterSwitch'.
+     */
+    protected void transitionToMasterSwitch(long dpid) {
+        synchronized (multiCacheLock) {
+            IOFSwitch sw = activeEqualSwitches.remove(dpid);
+            if (sw == null) {
+                log.error("Transition to master called on sw {}, but switch "
+                        + "was not found in controller-cache", dpid);
+                return;
+            }
+            activeMasterSwitches.put(dpid, sw);
+        }
+    }
+
+
+    /**
+     * Called when this controller's role for a switch transitions to equal.
+     * For 1.0 switches, we internally refer to the role 'slave' as
+     * 'equal'.
+     */
+    protected void transitionToEqualSwitch(long dpid) {
+        synchronized (multiCacheLock) {
+            IOFSwitch sw = activeMasterSwitches.remove(dpid);
+            if (sw == null) {
+                log.error("Transition to equal called on sw {}, but switch "
+                        + "was not found in controller-cache", dpid);
+                return;
+            }
+            activeEqualSwitches.put(dpid, sw);
+        }
+
+    }
+
+    /**
+     * Clear all state in controller switch maps for a switch that has
+     * disconnected from the local controller. Also release control for
+     * that switch from the global repository. Notify switch listeners.
+     */
+    protected void removeConnectedSwitch(long dpid) {
+        releaseRegistryControl(dpid);
+        connectedSwitches.remove(dpid);
+        IOFSwitch sw = activeMasterSwitches.remove(dpid);
+        if (sw == null) {
+            sw = activeEqualSwitches.remove(dpid);
+        }
+        if (sw != null) {
+            sw.cancelAllStatisticsReplies();
+            sw.setConnected(false); // do we need this?
+        }
+        counters.switchDisconnected.updateCounterWithFlush();
+
+    }
+
+    /**
+     * Indicates that ports on the given switch have changed. Enqueue a
+     * switch update.
+     * @param sw
+     */
+    protected void notifyPortChanged(long dpid, OFPortDesc port,
+            PortChangeType changeType) {
+        if (port == null || changeType == null) {
+            String msg = String.format("Switch port or changetType must not "
+                    + "be null in port change notification");
+            throw new NullPointerException(msg);
+        }
+        if (connectedSwitches.get(dpid) == null || getSwitch(dpid) == null) {
+            log.warn("Port change update on switch {} not connected or activated "
+                    + "... Aborting.", HexString.toHexString(dpid));
+            return;
+        }
+
+    }
+
+    // ***************
+    // Getters/Setters
+    // ***************
+
+
+    public synchronized void setIOFSwitchManager(IOFSwitchManager swManager) {
+        this.switchManager = swManager;
+        this.registryService = swManager.getRegistry();
+    }
+
+
+    public void setDebugCounter(IDebugCounterService dcs) {
+        this.debugCounters = dcs;
+    }
+
+    IDebugCounterService getDebugCounter() {
+        return this.debugCounters;
+    }
+
+    // **********************
+    // Role Handling
+    // **********************
+
+    /**
+     * created by ONOS - works with registry service.
+     */
+    protected class RoleChangeCallback implements ControlChangeCallback {
+        @Override
+        public void controlChanged(long dpidLong, boolean hasControl) {
+            Dpid dpid = new Dpid(dpidLong);
+            log.info("Role change callback for switch {}, hasControl {}",
+                    dpid, hasControl);
+
+            Role role = null;
+
+            /*
+             * issue #229
+             * Cannot rely on sw.getRole() as it can be behind due to pending
+             * role changes in the queue. Just submit it and late the
+             * RoleChanger handle duplicates.
+             */
+
+            if (hasControl) {
+                role = Role.MASTER;
+            } else {
+                role = Role.EQUAL; // treat the same as Role.SLAVE
+            }
+
+            OFChannelHandler swCh = connectedSwitches.get(dpid.value());
+            if (swCh == null) {
+                log.warn("Switch {} not found in connected switches", dpid);
+                return;
+            }
+
+            log.debug("Sending role request {} msg to {}", role, dpid);
+            swCh.sendRoleRequest(role, RoleRecvStatus.MATCHED_SET_ROLE);
+        }
+    }
+
+    /**
+     * Submit request to the registry service for mastership of the
+     * switch.
+     * @param dpid this datapath to get role for
+     */
+    public synchronized void submitRegistryRequest(long dpid) {
+        if (registryService == null) {
+            /*
+             * If we have no registry then simply assign
+             * mastership to this controller.
+             */
+            new RoleChangeCallback().controlChanged(dpid, true);
+            return;
+        }
+        OFChannelHandler h = connectedSwitches.get(dpid);
+        if (h == null) {
+            log.error("Trying to request registry control for switch {} "
+                    + "not in connected switches. Aborting.. ",
+                    HexString.toHexString(dpid));
+            connectedSwitches.get(dpid).disconnectSwitch();
+            return;
+        }
+        //Request control of the switch from the global registry
+        try {
+            h.controlRequested = Boolean.TRUE;
+            registryService.requestControl(dpid, new RoleChangeCallback());
+        } catch (RegistryException e) {
+            log.debug("Registry error: {}", e.getMessage());
+            h.controlRequested = Boolean.FALSE;
+        }
+        if (!h.controlRequested) { // XXX what is being attempted here?
+            // yield to allow other thread(s) to release control
+            // TODO AAS: this is awful and needs to be fixed
+            Thread.yield();
+            // safer to bounce the switch to reconnect here than proceeding further
+            // XXX S why? can't we just try again a little later?
+            log.debug("Closing sw:{} because we weren't able to request control " +
+                    "successfully" + dpid);
+            connectedSwitches.get(dpid).disconnectSwitch();
+        }
+    }
+
+    /**
+     * Relinquish role for the switch.
+     * @param dpidLong the controlled datapath
+     */
+    public synchronized void releaseRegistryControl(long dpidLong) {
+        OFChannelHandler h = connectedSwitches.get(dpidLong);
+        if (h == null) {
+            log.error("Trying to release registry control for switch {} "
+                    + "not in connected switches. Aborting.. ",
+                    HexString.toHexString(dpidLong));
+            return;
+        }
+        if (h.controlRequested) {
+            registryService.releaseControl(dpidLong);
+        }
+    }
+
+
+
+    // ***************
+    // IFloodlightProviderService
+    // ***************
+
+    // FIXME: remove this method
+    public Map<Long, IOFSwitch> getSwitches() {
+        return getMasterSwitches();
+    }
+
+    // FIXME: remove this method
+    public Map<Long, IOFSwitch> getMasterSwitches() {
+        return Collections.unmodifiableMap(activeMasterSwitches);
+    }
+
+
+
+    public Set<Long> getAllSwitchDpids() {
+        Set<Long> dpids = new HashSet<Long>();
+        dpids.addAll(activeMasterSwitches.keySet());
+        dpids.addAll(activeEqualSwitches.keySet());
+        return dpids;
+    }
+
+
+    public Set<Long> getAllMasterSwitchDpids() {
+        Set<Long> dpids = new HashSet<Long>();
+        dpids.addAll(activeMasterSwitches.keySet());
+        return dpids;
+    }
+
+
+    public Set<Long> getAllEqualSwitchDpids() {
+        Set<Long> dpids = new HashSet<Long>();
+        dpids.addAll(activeEqualSwitches.keySet());
+        return dpids;
+    }
+
+
+    public IOFSwitch getSwitch(long dpid) {
+        IOFSwitch sw = null;
+        sw = activeMasterSwitches.get(dpid);
+        if (sw != null) {
+            return sw;
+        }
+        sw = activeEqualSwitches.get(dpid);
+        if (sw != null) {
+            return sw;
+        }
+        return sw;
+    }
+
+
+    public IOFSwitch getMasterSwitch(long dpid) {
+        return  activeMasterSwitches.get(dpid);
+    }
+
+
+    public IOFSwitch getEqualSwitch(long dpid) {
+        return  activeEqualSwitches.get(dpid);
+    }
+
+
+
+
+
+    public OFFactory getOFMessageFactory10() {
+        return FACTORY10;
+    }
+
+
+    public OFFactory getOFMessageFactory13() {
+        return FACTORY13;
+    }
+
+
+
+    public Map<String, String> getControllerNodeIPs() {
+        // We return a copy of the mapping so we can guarantee that
+        // the mapping return is the same as one that will be (or was)
+        // dispatched to IHAListeners
+        HashMap<String, String> retval = new HashMap<String, String>();
+        synchronized (controllerNodeIPsCache) {
+            retval.putAll(controllerNodeIPsCache);
+        }
+        return retval;
+    }
+
+
+    public long getSystemStartTime() {
+        return (this.systemStartTime);
+    }
+
+
+    public void setAlwaysClearFlowsOnSwAdd(boolean value) {
+        this.alwaysClearFlowsOnSwAdd = value;
+    }
+
+
+    public InstanceId getInstanceId() {
+        return instanceId;
+    }
+
+
+    // **************
+    // Initialization
+    // **************
+
+    /**
+     * Tell controller that we're ready to accept switches loop.
+     *
+     * @throws IOException
+     */
+    @LogMessageDocs({
+            @LogMessageDoc(message = "Listening for switch connections on {address}",
+                    explanation = "The controller is ready and listening for new" +
+                            " switch connections"),
+            @LogMessageDoc(message = "Storage exception in controller " +
+                    "updates loop; terminating process",
+                    explanation = ERROR_DATABASE,
+                    recommendation = LogMessageDoc.CHECK_CONTROLLER),
+            @LogMessageDoc(level = "ERROR",
+                    message = "Exception in controller updates loop",
+                    explanation = "Failed to dispatch controller event",
+                    recommendation = LogMessageDoc.GENERIC_ACTION)
+    })
+    public void run() {
+
+        try {
+            final ServerBootstrap bootstrap = createServerBootStrap();
+
+            bootstrap.setOption("reuseAddr", true);
+            bootstrap.setOption("child.keepAlive", true);
+            bootstrap.setOption("child.tcpNoDelay", true);
+            bootstrap.setOption("child.sendBufferSize", Controller.SEND_BUFFER_SIZE);
+
+            ChannelPipelineFactory pfact =
+                    new OpenflowPipelineFactory(this, null);
+            bootstrap.setPipelineFactory(pfact);
+            InetSocketAddress sa = new InetSocketAddress(openFlowPort);
+            final ChannelGroup cg = new DefaultChannelGroup();
+            cg.add(bootstrap.bind(sa));
+
+            log.info("Listening for switch connections on {}", sa);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+    private ServerBootstrap createServerBootStrap() {
+        if (workerThreads == 0) {
+            return new ServerBootstrap(
+                    new NioServerSocketChannelFactory(
+                            Executors.newCachedThreadPool(),
+                            Executors.newCachedThreadPool()));
+        } else {
+            return new ServerBootstrap(
+                    new NioServerSocketChannelFactory(
+                            Executors.newCachedThreadPool(),
+                            Executors.newCachedThreadPool(), workerThreads));
+        }
+    }
+
+    public void setConfigParams(Map<String, String> configParams) {
+        String ofPort = configParams.get("openflowport");
+        if (ofPort != null) {
+            this.openFlowPort = Integer.parseInt(ofPort);
+        }
+        log.debug("OpenFlow port set to {}", this.openFlowPort);
+        String threads = configParams.get("workerthreads");
+        if (threads != null) {
+            this.workerThreads = Integer.parseInt(threads);
+        }
+        log.debug("Number of worker threads set to {}", this.workerThreads);
+        String controllerId = configParams.get("controllerid");
+        if (controllerId != null) {
+            this.instanceId = new InstanceId(controllerId);
+        } else {
+            //Try to get the hostname of the machine and use that for controller ID
+            try {
+                String hostname = java.net.InetAddress.getLocalHost().getHostName();
+                this.instanceId = new InstanceId(hostname);
+            } catch (UnknownHostException e) {
+                log.warn("Can't get hostname, using the default");
+            }
+        }
+
+        log.debug("ControllerId set to {}", this.instanceId);
+    }
+
+
+    /**
+     * Initialize internal data structures.
+     */
+    public void init(Map<String, String> configParams) {
+        // These data structures are initialized here because other
+        // module's startUp() might be called before ours
+        this.activeMasterSwitches = new ConcurrentHashMap<Long, IOFSwitch>();
+        this.activeEqualSwitches = new ConcurrentHashMap<Long, IOFSwitch>();
+        this.connectedSwitches = new ConcurrentHashMap<Long, OFChannelHandler>();
+        this.controllerNodeIPsCache = new HashMap<String, String>();
+
+        setConfigParams(configParams);
+        this.systemStartTime = System.currentTimeMillis();
+        this.setDebugCounter(new DebugCounter());
+        this.counters = new Counters();
+        this.multiCacheLock = new Object();
+
+
+        String option = configParams.get("flushSwitchesOnReconnect");
+        if (option != null && option.equalsIgnoreCase("true")) {
+            this.setAlwaysClearFlowsOnSwActivate(true);
+            log.info("Flush switches on reconnect -- Enabled.");
+        } else {
+            this.setAlwaysClearFlowsOnSwActivate(false);
+            log.info("Flush switches on reconnect -- Disabled");
+        }
+    }
+
+    /**
+     * Startup all of the controller's components.
+     */
+    @LogMessageDoc(message = "Waiting for storage source",
+            explanation = "The system database is not yet ready",
+            recommendation = "If this message persists, this indicates " +
+                    "that the system database has failed to start. " +
+                    LogMessageDoc.CHECK_CONTROLLER)
+    public synchronized void startupComponents() {
+        try {
+            if (registryService != null) {
+                registryService.registerController(instanceId.toString());
+            }
+        } catch (RegistryException e) {
+            log.warn("Registry service error: {}", e.getMessage());
+        }
+
+        // register counters and events
+        try {
+            this.counters.createCounters(debugCounters);
+        } catch (CounterException e) {
+            log.warn("Counters unavailable: {}", e.getMessage());
+        }
+    }
+
+    // **************
+    // debugCounter registrations
+    // **************
+
+    public static class Counters {
+        public static final String PREFIX = "controller";
+        public IDebugCounter switchActivated;
+        public IDebugCounter switchWithSameDpidActivated; // warn
+        public IDebugCounter switchDisconnected;
+        public IDebugCounter messageReceived;
+        public IDebugCounter switchDisconnectReadTimeout;
+        public IDebugCounter switchDisconnectHandshakeTimeout;
+        public IDebugCounter switchDisconnectIOError;
+        public IDebugCounter switchDisconnectParseError;
+        public IDebugCounter switchDisconnectSwitchStateException;
+        public IDebugCounter rejectedExecutionException;
+        public IDebugCounter switchDisconnectOtherException;
+        public IDebugCounter switchConnected;
+        public IDebugCounter unhandledMessage;
+        public IDebugCounter packetInWhileSwitchIsSlave;
+        public IDebugCounter epermErrorWhileSwitchIsMaster;
+        public IDebugCounter roleReplyTimeout;
+        public IDebugCounter roleReplyReceived; // expected RoleReply received
+        public IDebugCounter roleReplyErrorUnsupported;
+        public IDebugCounter switchCounterRegistrationFailed;
+
+        void createCounters(IDebugCounterService debugCounters) throws CounterException {
+
+            switchActivated =
+                debugCounters.registerCounter(
+                            PREFIX, "switch-activated",
+                            "A switch connected to this controller is now " +
+                            "in MASTER role",
+                            CounterType.ALWAYS_COUNT);
+
+            switchWithSameDpidActivated = // warn
+                debugCounters.registerCounter(
+                            PREFIX, "switch-with-same-dpid-activated",
+                            "A switch with the same DPID as another switch " +
+                            "connected to the controller. This can be " +
+                            "caused by multiple switches configured with " +
+                            "the same DPID or by a switch reconnecting very " +
+                            "quickly.",
+                            CounterType.COUNT_ON_DEMAND,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            switchDisconnected =
+                debugCounters.registerCounter(
+                            PREFIX, "switch-disconnected",
+                            "FIXME: switch has disconnected",
+                            CounterType.ALWAYS_COUNT);
+
+        //------------------------
+        // channel handler counters. Factor them out ??
+            messageReceived =
+                debugCounters.registerCounter(
+                            PREFIX, "message-received",
+                            "Number of OpenFlow messages received. Some of " +
+                            "these might be throttled",
+                            CounterType.ALWAYS_COUNT);
+
+            switchDisconnectReadTimeout =
+                debugCounters.registerCounter(
+                            PREFIX, "switch-disconnect-read-timeout",
+                            "Number of times a switch was disconnected due " +
+                            "due the switch failing to send OpenFlow " +
+                            "messages or responding to OpenFlow ECHOs",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+            switchDisconnectHandshakeTimeout =
+                debugCounters.registerCounter(
+                            PREFIX, "switch-disconnect-handshake-timeout",
+                            "Number of times a switch was disconnected " +
+                            "because it failed to complete the handshake " +
+                            "in time.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+            switchDisconnectIOError =
+                debugCounters.registerCounter(
+                            PREFIX, "switch-disconnect-io-error",
+                            "Number of times a switch was disconnected " +
+                            "due to IO errors on the switch connection.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+            switchDisconnectParseError =
+                debugCounters.registerCounter(
+                            PREFIX, "switch-disconnect-parse-error",
+                           "Number of times a switch was disconnected " +
+                           "because it sent an invalid packet that could " +
+                           "not be parsed",
+                           CounterType.ALWAYS_COUNT,
+                           IDebugCounterService.CTR_MDATA_ERROR);
+
+            switchDisconnectSwitchStateException =
+                debugCounters.registerCounter(
+                            PREFIX, "switch-disconnect-switch-state-exception",
+                            "Number of times a switch was disconnected " +
+                            "because it sent messages that were invalid " +
+                            "given the switch connection's state.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+            rejectedExecutionException =
+                debugCounters.registerCounter(
+                            PREFIX, "rejected-execution-exception",
+                            "TODO",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+
+            switchDisconnectOtherException =
+                debugCounters.registerCounter(
+                            PREFIX,  "switch-disconnect-other-exception",
+                            "Number of times a switch was disconnected " +
+                            "due to an exceptional situation not covered " +
+                            "by other counters",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+
+            switchConnected =
+                debugCounters.registerCounter(
+                            PREFIX, "switch-connected",
+                            "Number of times a new switch connection was " +
+                            "established",
+                            CounterType.ALWAYS_COUNT);
+
+            unhandledMessage =
+                debugCounters.registerCounter(
+                            PREFIX, "unhandled-message",
+                            "Number of times an OpenFlow message was " +
+                            "received that the controller ignored because " +
+                            "it was inapproriate given the switch " +
+                            "connection's state.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+                            // might be less than warning
+
+            packetInWhileSwitchIsSlave =
+                debugCounters.registerCounter(
+                            PREFIX, "packet-in-while-switch-is-slave",
+                            "Number of times a packet in was received " +
+                            "from a switch that was in SLAVE role. " +
+                            "Possibly inidicates inconsistent roles.",
+                            CounterType.ALWAYS_COUNT);
+            epermErrorWhileSwitchIsMaster =
+                debugCounters.registerCounter(
+                            PREFIX, "eperm-error-while-switch-is-master",
+                            "Number of times a permission error was " +
+                            "received while the switch was in MASTER role. " +
+                            "Possibly inidicates inconsistent roles.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            roleReplyTimeout =
+                debugCounters.registerCounter(
+                            PREFIX, "role-reply-timeout",
+                            "Number of times a role request message did not " +
+                            "receive the expected reply from a switch",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            roleReplyReceived = // expected RoleReply received
+                debugCounters.registerCounter(
+                            PREFIX, "role-reply-received",
+                            "Number of times the controller received the " +
+                            "expected role reply message from a switch",
+                            CounterType.ALWAYS_COUNT);
+
+            roleReplyErrorUnsupported =
+                debugCounters.registerCounter(
+                            PREFIX, "role-reply-error-unsupported",
+                            "Number of times the controller received an " +
+                            "error from a switch in response to a role " +
+                            "request indicating that the switch does not " +
+                            "support roles.",
+                            CounterType.ALWAYS_COUNT);
+
+            switchCounterRegistrationFailed =
+                    debugCounters.registerCounter(PREFIX,
+                                "switch-counter-registration-failed",
+                                "Number of times the controller failed to " +
+                                "register per-switch debug counters",
+                                CounterType.ALWAYS_COUNT,
+                                IDebugCounterService.CTR_MDATA_WARN);
+
+
+        }
+    }
+
+    public Counters getCounters() {
+        return this.counters;
+    }
+
+
+    // **************
+    // Utility methods
+    // **************
+
+
+    public void setAlwaysClearFlowsOnSwActivate(boolean value) {
+        //this.alwaysClearFlowsOnSwActivate = value;
+        // XXX S need to be a little more careful about this
+    }
+
+    public Map<String, Long> getMemory() {
+        Map<String, Long> m = new HashMap<String, Long>();
+        Runtime runtime = Runtime.getRuntime();
+        m.put("total", runtime.totalMemory());
+        m.put("free", runtime.freeMemory());
+        return m;
+    }
+
+
+    public Long getUptime() {
+        RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
+        return rb.getUptime();
+    }
+
+    /**
+     * Forward to the driver-manager to get an IOFSwitch instance.
+     * @param desc
+     * @return
+     */
+    protected IOFSwitch getOFSwitchInstance(OFDescStatsReply desc, OFVersion ofv) {
+        if (switchManager == null) {
+            return new DummySwitchForTesting();
+        }
+        return switchManager.getSwitchImpl(desc.getMfrDesc(), desc.getHwDesc(),
+                                            desc.getSwDesc(), ofv);
+    }
+
+    @Activate
+    public void activate() {
+        log.info("Initialising OpenFlow Lib and IO");
+        this.init(new HashMap<String, String>());
+        this.startupComponents();
+        this.run();
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/HandshakeTimeoutException.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/HandshakeTimeoutException.java
new file mode 100644
index 0000000..48856a9
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/HandshakeTimeoutException.java
@@ -0,0 +1,29 @@
+/**
+ *    Copyright 2011, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.internal;
+
+/**
+ * Exception is thrown when the handshake fails to complete.
+ * before a specified time
+ *
+ */
+public class HandshakeTimeoutException extends Exception {
+
+    private static final long serialVersionUID = 6859880268940337312L;
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/HandshakeTimeoutHandler.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/HandshakeTimeoutHandler.java
new file mode 100644
index 0000000..2ed3fd2
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/HandshakeTimeoutHandler.java
@@ -0,0 +1,94 @@
+/**
+*    Copyright 2011, Big Switch Networks, Inc.
+*    Originally created by David Erickson, Stanford University
+*
+*    Licensed under the Apache License, Version 2.0 (the "License"); you may
+*    not use this file except in compliance with the License. You may obtain
+*    a copy of the License at
+*
+*         http://www.apache.org/licenses/LICENSE-2.0
+*
+*    Unless required by applicable law or agreed to in writing, software
+*    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+*    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+*    License for the specific language governing permissions and limitations
+*    under the License.
+**/
+
+package net.onrc.onos.of.ctl.internal;
+
+import java.util.concurrent.TimeUnit;
+
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
+import org.jboss.netty.util.Timeout;
+import org.jboss.netty.util.Timer;
+import org.jboss.netty.util.TimerTask;
+
+/**
+ * Trigger a timeout if a switch fails to complete handshake soon enough.
+ */
+public class HandshakeTimeoutHandler
+    extends SimpleChannelUpstreamHandler {
+    static final HandshakeTimeoutException EXCEPTION =
+            new HandshakeTimeoutException();
+
+    final OFChannelHandler channelHandler;
+    final Timer timer;
+    final long timeoutNanos;
+    volatile Timeout timeout;
+
+    public HandshakeTimeoutHandler(OFChannelHandler channelHandler,
+                                   Timer timer,
+                                   long timeoutSeconds) {
+        super();
+        this.channelHandler = channelHandler;
+        this.timer = timer;
+        this.timeoutNanos = TimeUnit.SECONDS.toNanos(timeoutSeconds);
+
+    }
+
+    @Override
+    public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e)
+            throws Exception {
+        if (timeoutNanos > 0) {
+            timeout = timer.newTimeout(new HandshakeTimeoutTask(ctx),
+                                       timeoutNanos, TimeUnit.NANOSECONDS);
+        }
+        ctx.sendUpstream(e);
+    }
+
+    @Override
+    public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e)
+            throws Exception {
+        if (timeout != null) {
+            timeout.cancel();
+            timeout = null;
+        }
+    }
+
+    private final class HandshakeTimeoutTask implements TimerTask {
+
+        private final ChannelHandlerContext ctx;
+
+        HandshakeTimeoutTask(ChannelHandlerContext ctx) {
+            this.ctx = ctx;
+        }
+
+        @Override
+        public void run(Timeout t) throws Exception {
+            if (t.isCancelled()) {
+                return;
+            }
+
+            if (!ctx.getChannel().isOpen()) {
+                return;
+            }
+            if (!channelHandler.isHandshakeComplete()) {
+                Channels.fireExceptionCaught(ctx, EXCEPTION);
+            }
+        }
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFChannelHandler.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFChannelHandler.java
new file mode 100644
index 0000000..40de530
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFChannelHandler.java
@@ -0,0 +1,2153 @@
+package net.onrc.onos.of.ctl.internal;
+
+import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.RejectedExecutionException;
+
+import net.onrc.onos.of.ctl.IOFSwitch;
+import net.onrc.onos.of.ctl.IOFSwitch.PortChangeEvent;
+import net.onrc.onos.of.ctl.Role;
+import net.onrc.onos.of.ctl.annotations.LogMessageDoc;
+import net.onrc.onos.of.ctl.annotations.LogMessageDocs;
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounterService.CounterException;
+import net.onrc.onos.of.ctl.internal.Controller.Counters;
+import net.onrc.onos.of.ctl.internal.OFChannelHandler.ChannelState.RoleReplyInfo;
+
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
+import org.jboss.netty.handler.timeout.IdleStateEvent;
+import org.jboss.netty.handler.timeout.ReadTimeoutException;
+import org.projectfloodlight.openflow.exceptions.OFParseError;
+import org.projectfloodlight.openflow.protocol.OFAsyncGetReply;
+import org.projectfloodlight.openflow.protocol.OFBadRequestCode;
+import org.projectfloodlight.openflow.protocol.OFBarrierReply;
+import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
+import org.projectfloodlight.openflow.protocol.OFControllerRole;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFDescStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFEchoReply;
+import org.projectfloodlight.openflow.protocol.OFEchoRequest;
+import org.projectfloodlight.openflow.protocol.OFErrorMsg;
+import org.projectfloodlight.openflow.protocol.OFErrorType;
+import org.projectfloodlight.openflow.protocol.OFExperimenter;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFFeaturesReply;
+import org.projectfloodlight.openflow.protocol.OFFlowModFailedCode;
+import org.projectfloodlight.openflow.protocol.OFFlowRemoved;
+import org.projectfloodlight.openflow.protocol.OFGetConfigReply;
+import org.projectfloodlight.openflow.protocol.OFGetConfigRequest;
+import org.projectfloodlight.openflow.protocol.OFHello;
+import org.projectfloodlight.openflow.protocol.OFHelloElem;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFNiciraControllerRole;
+import org.projectfloodlight.openflow.protocol.OFNiciraControllerRoleReply;
+import org.projectfloodlight.openflow.protocol.OFPacketIn;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFPortStatus;
+import org.projectfloodlight.openflow.protocol.OFQueueGetConfigReply;
+import org.projectfloodlight.openflow.protocol.OFRoleReply;
+import org.projectfloodlight.openflow.protocol.OFRoleRequest;
+import org.projectfloodlight.openflow.protocol.OFSetConfig;
+import org.projectfloodlight.openflow.protocol.OFStatsReply;
+import org.projectfloodlight.openflow.protocol.OFStatsReplyFlags;
+import org.projectfloodlight.openflow.protocol.OFStatsType;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.protocol.errormsg.OFBadRequestErrorMsg;
+import org.projectfloodlight.openflow.protocol.errormsg.OFFlowModFailedErrorMsg;
+import org.projectfloodlight.openflow.protocol.errormsg.OFRoleRequestFailedErrorMsg;
+import org.projectfloodlight.openflow.types.U32;
+import org.projectfloodlight.openflow.types.U64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/**
+ * Channel handler deals with the switch connection and dispatches
+ * switch messages to the appropriate locations.
+ */
+class OFChannelHandler extends IdleStateAwareChannelHandler {
+    private static final Logger log = LoggerFactory.getLogger(OFChannelHandler.class);
+    private static final long DEFAULT_ROLE_TIMEOUT_MS = 2 * 1000; // 10 sec
+    private final Controller controller;
+    private final Counters counters;
+    private IOFSwitch sw;
+    private long thisdpid; // channelHandler cached value of connected switch id
+    private Channel channel;
+    // State needs to be volatile because the HandshakeTimeoutHandler
+    // needs to check if the handshake is complete
+    private volatile ChannelState state;
+
+    // All role messaging is handled by the roleChanger. The channel state machine
+    // coordinates between the roleChanger and the controller-global-registry-service
+    // to determine controller roles per switch.
+    private RoleChanger roleChanger;
+    // Used to coordinate between the controller and the cleanup thread(?)
+    // for access to the global registry on a per switch basis.
+    volatile Boolean controlRequested;
+    // When a switch with a duplicate dpid is found (i.e we already have a
+    // connected switch with the same dpid), the new switch is immediately
+    // disconnected. At that point netty callsback channelDisconnected() which
+    // proceeds to cleaup switch state - we need to ensure that it does not cleanup
+    // switch state for the older (still connected) switch
+    private volatile Boolean duplicateDpidFound;
+
+    // Temporary storage for switch-features and port-description
+    private OFFeaturesReply featuresReply;
+    private OFPortDescStatsReply portDescReply;
+    // a concurrent ArrayList to temporarily store port status messages
+    // before we are ready to deal with them
+    private final CopyOnWriteArrayList<OFPortStatus> pendingPortStatusMsg;
+
+    //Indicates the openflow version used by this switch
+    protected OFVersion ofVersion;
+    protected OFFactory factory13;
+    protected OFFactory factory10;
+
+    /** transaction Ids to use during handshake. Since only one thread
+     * calls into an OFChannelHandler instance, we don't need atomic.
+     * We will count down
+     */
+    private int handshakeTransactionIds = -1;
+
+    /**
+     * Create a new unconnected OFChannelHandler.
+     * @param controller
+     */
+    OFChannelHandler(Controller controller) {
+        this.controller = controller;
+        this.counters = controller.getCounters();
+        this.roleChanger = new RoleChanger(DEFAULT_ROLE_TIMEOUT_MS);
+        this.state = ChannelState.INIT;
+        this.pendingPortStatusMsg = new CopyOnWriteArrayList<OFPortStatus>();
+        factory13 = controller.getOFMessageFactory13();
+        factory10 = controller.getOFMessageFactory10();
+        controlRequested = Boolean.FALSE;
+        duplicateDpidFound = Boolean.FALSE;
+    }
+
+    //*******************
+    //  Role Handling
+    //*******************
+
+    /**
+     * When we remove a pending role request we use this enum to indicate how we
+     * arrived at the decision. When we send a role request to the switch, we
+     * also use  this enum to indicate what we expect back from the switch, so the
+     * role changer can match the reply to our expectation.
+     */
+    public enum RoleRecvStatus {
+        /** The switch returned an error indicating that roles are not.
+         * supported*/
+        UNSUPPORTED,
+        /** The request timed out. */
+        NO_REPLY,
+        /** The reply was old, there is a newer request pending. */
+        OLD_REPLY,
+        /**
+         *  The reply's role matched the role that this controller set in the
+         *  request message - invoked either initially at startup or to reassert
+         *  current role.
+         */
+        MATCHED_CURRENT_ROLE,
+        /**
+         *  The reply's role matched the role that this controller set in the
+         *  request message - this is the result of a callback from the
+         *  global registry, followed by a role request sent to the switch.
+         */
+        MATCHED_SET_ROLE,
+        /**
+         * The reply's role was a response to the query made by this controller.
+         */
+        REPLY_QUERY,
+        /** We received a role reply message from the switch
+         *  but the expectation was unclear, or there was no expectation.
+         */
+        OTHER_EXPECTATION,
+    }
+
+    /**
+     * Forwards to RoleChanger. See there.
+     * @param role
+     */
+    public void sendRoleRequest(Role role, RoleRecvStatus expectation) {
+        try {
+            roleChanger.sendRoleRequest(role, expectation);
+        } catch (IOException e) {
+            log.error("Disconnecting switch {} due to IO Error: {}",
+                    getSwitchInfoString(), e.getMessage());
+            channel.close();
+        }
+    }
+
+    // XXX S consider if necessary
+    public void disconnectSwitch() {
+        sw.disconnectSwitch();
+    }
+
+    /**
+     * A utility class to handle role requests and replies for this channel.
+     * After a role request is submitted the role changer keeps track of the
+     * pending request, collects the reply (if any) and times out the request
+     * if necessary.
+     *
+     * To simplify role handling we only keep track of the /last/ pending
+     * role reply send to the switch. If multiple requests are pending and
+     * we receive replies for earlier requests we ignore them. However, this
+     * way of handling pending requests implies that we could wait forever if
+     * a new request is submitted before the timeout triggers. If necessary
+     * we could work around that though.
+     */
+    private class RoleChanger {
+        // indicates that a request is currently pending
+        // needs to be volatile to allow correct double-check idiom
+        private volatile boolean requestPending;
+        // the transaction Id of the pending request
+        private int pendingXid;
+        // the role that's pending
+        private Role pendingRole;
+        // system time in MS when we send the request
+        private long roleSubmitTime;
+        // the timeout to use
+        private final long roleTimeoutMs;
+        // the expectation set by the caller for the returned role
+        private RoleRecvStatus expectation;
+
+        public RoleChanger(long roleTimeoutMs) {
+            this.requestPending = false;
+            this.roleSubmitTime = 0;
+            this.pendingXid = -1;
+            this.pendingRole = null;
+            this.roleTimeoutMs = roleTimeoutMs;
+            this.expectation = RoleRecvStatus.MATCHED_CURRENT_ROLE;
+        }
+
+        /**
+         * Send NX role request message to the switch requesting the specified
+         * role.
+         *
+         * @param sw switch to send the role request message to
+         * @param role role to request
+         */
+        private int sendNxRoleRequest(Role role) throws IOException {
+            // Convert the role enum to the appropriate role to send
+            OFNiciraControllerRole roleToSend = OFNiciraControllerRole.ROLE_OTHER;
+            switch (role) {
+            case MASTER:
+                roleToSend = OFNiciraControllerRole.ROLE_MASTER;
+                break;
+            case SLAVE:
+            case EQUAL:
+            default:
+                // ensuring that the only two roles sent to 1.0 switches with
+                // Nicira role support, are MASTER and SLAVE
+                roleToSend = OFNiciraControllerRole.ROLE_SLAVE;
+                log.warn("Sending Nx Role.SLAVE to switch {}.", sw);
+            }
+            int xid = sw.getNextTransactionId();
+            OFExperimenter roleRequest = factory10
+                    .buildNiciraControllerRoleRequest()
+                    .setXid(xid)
+                    .setRole(roleToSend)
+                    .build();
+            sw.write(Collections.<OFMessage>singletonList(roleRequest));
+            return xid;
+        }
+
+        private int sendOF13RoleRequest(Role role) throws IOException {
+            // Convert the role enum to the appropriate role to send
+            OFControllerRole roleToSend = OFControllerRole.ROLE_NOCHANGE;
+            switch (role) {
+            case EQUAL:
+                roleToSend = OFControllerRole.ROLE_EQUAL;
+                break;
+            case MASTER:
+                roleToSend = OFControllerRole.ROLE_MASTER;
+                break;
+            case SLAVE:
+                roleToSend = OFControllerRole.ROLE_SLAVE;
+                break;
+            default:
+                log.warn("Sending default role.noChange to switch {}."
+                        + " Should only be used for queries.", sw);
+            }
+
+            int xid = sw.getNextTransactionId();
+            OFRoleRequest rrm = factory13
+                    .buildRoleRequest()
+                    .setRole(roleToSend)
+                    .setXid(xid)
+                    .setGenerationId(sw.getNextGenerationId())
+                    .build();
+            sw.write(rrm);
+            return xid;
+        }
+
+        /**
+         * Send a role request with the given role to the switch and update
+         * the pending request and timestamp.
+         * Sends an OFPT_ROLE_REQUEST to an OF1.3 switch, OR
+         * Sends an NX_ROLE_REQUEST to an OF1.0 switch if configured to support it
+         * in the IOFSwitch driver. If not supported, this method sends nothing
+         * and returns 'false'. The caller should take appropriate action.
+         *
+         * One other optimization we do here is that for OF1.0 switches with
+         * Nicira role message support, we force the Role.EQUAL to become
+         * Role.SLAVE, as there is no defined behavior for the Nicira role OTHER.
+         * We cannot expect it to behave like SLAVE. We don't have this problem with
+         * OF1.3 switches, because Role.EQUAL is well defined and we can simulate
+         * SLAVE behavior by using ASYNC messages.
+         *
+         * @param role
+         * @throws IOException
+         * @returns false if and only if the switch does not support role-request
+         * messages, according to the switch driver; true otherwise.
+         */
+        synchronized boolean sendRoleRequest(Role role, RoleRecvStatus exp)
+                throws IOException {
+            this.expectation = exp;
+
+            if (ofVersion == OFVersion.OF_10) {
+                Boolean supportsNxRole = (Boolean)
+                        sw.getAttribute(IOFSwitch.SWITCH_SUPPORTS_NX_ROLE);
+                if (!supportsNxRole) {
+                    log.debug("Switch driver indicates no support for Nicira "
+                            + "role request messages. Not sending ...");
+                    state.handleUnsentRoleMessage(OFChannelHandler.this, role,
+                            expectation);
+                    return false;
+                }
+                // OF1.0 switch with support for NX_ROLE_REQUEST vendor extn.
+                // make Role.EQUAL become Role.SLAVE
+                role = (role == Role.EQUAL) ? Role.SLAVE : role;
+                pendingXid = sendNxRoleRequest(role);
+                pendingRole = role;
+                roleSubmitTime = System.currentTimeMillis();
+                requestPending = true;
+            } else {
+                // OF1.3 switch, use OFPT_ROLE_REQUEST message
+                pendingXid = sendOF13RoleRequest(role);
+                pendingRole = role;
+                roleSubmitTime = System.currentTimeMillis();
+                requestPending = true;
+            }
+            return true;
+        }
+
+        /**
+         * Deliver a received role reply.
+         *
+         * Check if a request is pending and if the received reply matches the
+         * the expected pending reply (we check both role and xid) we set
+         * the role for the switch/channel.
+         *
+         * If a request is pending but doesn't match the reply we ignore it, and
+         * return
+         *
+         * If no request is pending we disconnect with a SwitchStateException
+         *
+         * @param RoleReplyInfo information about role-reply in format that
+         *                      controller can understand.
+         * @throws SwitchStateException if no request is pending
+         */
+        synchronized RoleRecvStatus deliverRoleReply(RoleReplyInfo rri)
+                throws SwitchStateException {
+            if (!requestPending) {
+                Role currentRole = (sw != null) ? sw.getRole() : null;
+                if (currentRole != null) {
+                    if (currentRole == rri.getRole()) {
+                        // Don't disconnect if the role reply we received is
+                        // for the same role we are already in.
+                        log.debug("Received unexpected RoleReply from "
+                                + "Switch: {} in State: {}. "
+                                + "Role in reply is same as current role of this "
+                                + "controller for this sw. Ignoring ...",
+                                getSwitchInfoString(), state.toString());
+                        return RoleRecvStatus.OTHER_EXPECTATION;
+                    } else {
+                        String msg = String.format("Switch: [%s], State: [%s], "
+                                + "received unexpected RoleReply[%s]. "
+                                + "No roles are pending, and this controller's "
+                                + "current role:[%s] does not match reply. "
+                                + "Disconnecting switch ... ",
+                                OFChannelHandler.this.getSwitchInfoString(),
+                                OFChannelHandler.this.state.toString(),
+                                rri, currentRole);
+                        throw new SwitchStateException(msg);
+                    }
+                }
+                log.debug("Received unexpected RoleReply {} from "
+                        + "Switch: {} in State: {}. "
+                        + "This controller has no current role for this sw. "
+                        + "Ignoring ...", new Object[] {rri,
+                                getSwitchInfoString(), state});
+                return RoleRecvStatus.OTHER_EXPECTATION;
+            }
+
+            int xid = (int) rri.getXid();
+            Role role = rri.getRole();
+            // XXX S should check generation id meaningfully and other cases of expectations
+            // U64 genId = rri.getGenId();
+
+            if (pendingXid != xid) {
+                log.debug("Received older role reply from " +
+                        "switch {} ({}). Ignoring. " +
+                        "Waiting for {}, xid={}",
+                        new Object[] {getSwitchInfoString(), rri,
+                        pendingRole, pendingXid });
+                return RoleRecvStatus.OLD_REPLY;
+            }
+
+            if (pendingRole == role) {
+                log.debug("Received role reply message from {} that matched "
+                        + "expected role-reply {} with expectations {}",
+                        new Object[] {getSwitchInfoString(), role, expectation});
+                counters.roleReplyReceived.updateCounterWithFlush();
+                //setSwitchRole(role, RoleRecvStatus.RECEIVED_REPLY); dont want to set state here
+                if (expectation == RoleRecvStatus.MATCHED_CURRENT_ROLE ||
+                        expectation == RoleRecvStatus.MATCHED_SET_ROLE) {
+                    return expectation;
+                } else {
+                    return RoleRecvStatus.OTHER_EXPECTATION;
+                }
+            }
+
+            // if xids match but role's don't, perhaps its a query (OF1.3)
+            if (expectation == RoleRecvStatus.REPLY_QUERY) {
+                return expectation;
+            }
+
+            return RoleRecvStatus.OTHER_EXPECTATION;
+        }
+
+        /**
+         * Called if we receive an  error message. If the xid matches the
+         * pending request we handle it otherwise we ignore it.
+         *
+         * Note: since we only keep the last pending request we might get
+         * error messages for earlier role requests that we won't be able
+         * to handle
+         */
+        synchronized RoleRecvStatus deliverError(OFErrorMsg error)
+                throws SwitchStateException {
+            if (!requestPending) {
+                log.debug("Received an error msg from sw {}, but no pending "
+                        + "requests in role-changer; not handling ...",
+                        getSwitchInfoString());
+                return RoleRecvStatus.OTHER_EXPECTATION;
+            }
+            if (pendingXid != error.getXid()) {
+                if (error.getErrType() == OFErrorType.ROLE_REQUEST_FAILED) {
+                    log.debug("Received an error msg from sw {} for a role request,"
+                            + " but not for pending request in role-changer; "
+                            + " ignoring error {} ...",
+                            getSwitchInfoString(), error);
+                }
+                return RoleRecvStatus.OTHER_EXPECTATION;
+            }
+            // it is an error related to a currently pending role request message
+            if (error.getErrType() == OFErrorType.BAD_REQUEST) {
+                counters.roleReplyErrorUnsupported.updateCounterWithFlush();
+                log.error("Received a error msg {} from sw {} in state {} for "
+                        + "pending role request {}. Switch driver indicates "
+                        + "role-messaging is supported. Possible issues in "
+                        + "switch driver configuration?", new Object[] {
+                                ((OFBadRequestErrorMsg) error).toString(),
+                                getSwitchInfoString(), state, pendingRole
+                        });
+                return RoleRecvStatus.UNSUPPORTED;
+            }
+
+            if (error.getErrType() == OFErrorType.ROLE_REQUEST_FAILED) {
+                OFRoleRequestFailedErrorMsg rrerr =
+                        (OFRoleRequestFailedErrorMsg) error;
+                switch (rrerr.getCode()) {
+                case BAD_ROLE:
+                    // switch says that current-role-req has bad role?
+                    // for now we disconnect
+                    // fall-thru
+                case STALE:
+                    // switch says that current-role-req has stale gen-id?
+                    // for now we disconnect
+                    // fall-thru
+                case UNSUP:
+                    // switch says that current-role-req has role that
+                    // cannot be supported? for now we disconnect
+                    String msgx = String.format("Switch: [%s], State: [%s], "
+                            + "received Error to for pending role request [%s]. "
+                            + "Error:[%s]. Disconnecting switch ... ",
+                            OFChannelHandler.this.getSwitchInfoString(),
+                            OFChannelHandler.this.state.toString(),
+                            pendingRole, rrerr);
+                    throw new SwitchStateException(msgx);
+                default:
+                    break;
+                }
+            }
+
+            // This error message was for a role request message but we dont know
+            // how to handle errors for nicira role request messages
+            return RoleRecvStatus.OTHER_EXPECTATION;
+        }
+
+        /**
+         * Check if a pending role request has timed out.
+         */
+        void checkTimeout() {
+            if (!requestPending) {
+                return;
+            }
+            synchronized (this) {
+                if (!requestPending) {
+                    return;
+                }
+                long now = System.currentTimeMillis();
+                if (now - roleSubmitTime > roleTimeoutMs) {
+                    // timeout triggered.
+                    counters.roleReplyTimeout.updateCounterWithFlush();
+                    //setSwitchRole(pendingRole, RoleRecvStatus.NO_REPLY);
+                    // XXX S come back to this
+                }
+            }
+        }
+
+    }
+
+    //*************************
+    //  Channel State Machine
+    //*************************
+
+    /**
+     * The state machine for handling the switch/channel state. All state
+     * transitions should happen from within the state machine (and not from other
+     * parts of the code)
+     */
+    enum ChannelState {
+        /**
+         * Initial state before channel is connected.
+         */
+        INIT(false) {
+            @Override
+            void processOFMessage(OFChannelHandler h, OFMessage m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException {
+                // need to implement since its abstract but it will never
+                // be called
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                unhandledMessageReceived(h, m);
+            }
+        },
+
+        /**
+         * We send a OF 1.3 HELLO to the switch and wait for a Hello from the switch.
+         * Once we receive the reply, we decide on OF 1.3 or 1.0 switch - no other
+         * protocol version is accepted.
+         * We send an OFFeaturesRequest depending on the protocol version selected
+         * Next state is WAIT_FEATURES_REPLY
+         */
+        WAIT_HELLO(false) {
+            @Override
+            void processOFHello(OFChannelHandler h, OFHello m)
+                    throws IOException {
+                // TODO We could check for the optional bitmap, but for now
+                // we are just checking the version number.
+                if (m.getVersion() == OFVersion.OF_13) {
+                    log.info("Received {} Hello from {}", m.getVersion(),
+                            h.channel.getRemoteAddress());
+                    h.ofVersion = OFVersion.OF_13;
+                } else if (m.getVersion() == OFVersion.OF_10) {
+                    log.info("Received {} Hello from {} - switching to OF "
+                            + "version 1.0", m.getVersion(),
+                            h.channel.getRemoteAddress());
+                    h.ofVersion = OFVersion.OF_10;
+                } else {
+                    log.error("Received Hello of version {} from switch at {}. "
+                            + "This controller works with OF1.0 and OF1.3 "
+                            + "switches. Disconnecting switch ...",
+                            m.getVersion(), h.channel.getRemoteAddress());
+                    h.channel.disconnect();
+                    return;
+                }
+                h.sendHandshakeFeaturesRequestMessage();
+                h.setState(WAIT_FEATURES_REPLY);
+            }
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                    OFStatsReply  m)
+                            throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m) {
+                logErrorDisconnect(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                unhandledMessageReceived(h, m);
+            }
+        },
+
+
+        /**
+         * We are waiting for a features reply message. Once we receive it, the
+         * behavior depends on whether this is a 1.0 or 1.3 switch. For 1.0,
+         * we send a SetConfig request, barrier, and GetConfig request and the
+         * next state is WAIT_CONFIG_REPLY. For 1.3, we send a Port description
+         * request and the next state is WAIT_PORT_DESC_REPLY.
+         */
+        WAIT_FEATURES_REPLY(false) {
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException {
+                h.thisdpid = m.getDatapathId().getLong();
+                log.info("Received features reply for switch at {} with dpid {}",
+                        h.getSwitchInfoString(), h.thisdpid);
+                //update the controller about this connected switch
+                boolean success = h.controller.addConnectedSwitch(
+                        h.thisdpid, h);
+                if (!success) {
+                    disconnectDuplicate(h);
+                    return;
+                }
+
+                h.featuresReply = m; //temp store
+                if (h.ofVersion == OFVersion.OF_10) {
+                    h.sendHandshakeSetConfig();
+                    h.setState(WAIT_CONFIG_REPLY);
+                } else {
+                    //version is 1.3, must get switchport information
+                    h.sendHandshakeOFPortDescRequest();
+                    h.setState(WAIT_PORT_DESC_REPLY);
+                }
+            }
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                    OFStatsReply  m)
+                            throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m) {
+                logErrorDisconnect(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                unhandledMessageReceived(h, m);
+            }
+        },
+
+        /**
+         * We are waiting for a description of the 1.3 switch ports.
+         * Once received, we send a SetConfig request
+         * Next State is WAIT_CONFIG_REPLY
+         */
+        WAIT_PORT_DESC_REPLY(false) {
+
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h, OFStatsReply m)
+                    throws SwitchStateException {
+                // Read port description
+                if (m.getStatsType() != OFStatsType.PORT_DESC) {
+                    log.warn("Expecting port description stats but received stats "
+                            + "type {} from {}. Ignoring ...", m.getStatsType(),
+                            h.channel.getRemoteAddress());
+                    return;
+                }
+                if (m.getFlags().contains(OFStatsReplyFlags.REPLY_MORE)) {
+                    log.warn("Stats reply indicates more stats from sw {} for "
+                            + "port description - not currently handled",
+                            h.getSwitchInfoString());
+                }
+                h.portDescReply = (OFPortDescStatsReply) m; // temp store
+                log.info("Received port desc reply for switch at {}",
+                        h.getSwitchInfoString());
+                try {
+                    h.sendHandshakeSetConfig();
+                } catch (IOException e) {
+                    log.error("Unable to send setConfig after PortDescReply. "
+                            + "Error: {}", e.getMessage());
+                }
+                h.setState(WAIT_CONFIG_REPLY);
+            }
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException, SwitchStateException {
+                logErrorDisconnect(h, m);
+
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                unhandledMessageReceived(h, m);
+
+            }
+        },
+
+        /**
+         * We are waiting for a config reply message. Once we receive it
+         * we send a DescriptionStatsRequest to the switch.
+         * Next state: WAIT_DESCRIPTION_STAT_REPLY
+         */
+        WAIT_CONFIG_REPLY(false) {
+            @Override
+            @LogMessageDocs({
+                @LogMessageDoc(level = "WARN",
+                        message = "Config Reply from {switch} has "
+                                + "miss length set to {length}",
+                                explanation = "The controller requires that the switch "
+                                        + "use a miss length of 0xffff for correct "
+                                        + "function",
+                                        recommendation = "Use a different switch to ensure "
+                                                + "correct function")
+            })
+            void processOFGetConfigReply(OFChannelHandler h, OFGetConfigReply m)
+                    throws IOException {
+                if (m.getMissSendLen() == 0xffff) {
+                    log.trace("Config Reply from switch {} confirms "
+                            + "miss length set to 0xffff",
+                            h.getSwitchInfoString());
+                } else {
+                    // FIXME: we can't really deal with switches that don't send
+                    // full packets. Shouldn't we drop the connection here?
+                    log.warn("Config Reply from switch {} has"
+                            + "miss length set to {}",
+                            h.getSwitchInfoString(),
+                            m.getMissSendLen());
+                }
+                h.sendHandshakeDescriptionStatsRequest();
+                h.setState(WAIT_DESCRIPTION_STAT_REPLY);
+            }
+
+            @Override
+            void processOFBarrierReply(OFChannelHandler h, OFBarrierReply m) {
+                // do nothing;
+            }
+
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                    OFStatsReply  m)
+                            throws IOException, SwitchStateException {
+                log.error("Received multipart(stats) message sub-type {}",
+                        m.getStatsType());
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m) {
+                logErrorDisconnect(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                h.pendingPortStatusMsg.add(m);
+            }
+        },
+
+
+        /**
+         * We are waiting for a OFDescriptionStat message from the switch.
+         * Once we receive any stat message we try to parse it. If it's not
+         * a description stats message we disconnect. If its the expected
+         * description stats message, we:
+         *    - use the switch driver to bind the switch and get an IOFSwitch instance
+         *    - setup the IOFSwitch instance
+         *    - add switch to FloodlightProvider(Controller) and send the initial role
+         *      request to the switch.
+         * Next state: WAIT_INITIAL_ROLE
+         *      In the typical case, where switches support role request messages
+         *      the next state is where we expect the role reply message.
+         *      In the special case that where the switch does not support any kind
+         *      of role request messages, we don't send a role message, but we do
+         *      request mastership from the registry service. This controller
+         *      should become master once we hear back from the registry service.
+         * All following states will have a h.sw instance!
+         */
+        WAIT_DESCRIPTION_STAT_REPLY(false) {
+            @LogMessageDoc(message = "Switch {switch info} bound to class "
+                    + "{switch driver}, description {switch description}",
+                    explanation = "The specified switch has been bound to "
+                            + "a switch driver based on the switch description"
+                            + "received from the switch")
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h, OFStatsReply m)
+                    throws SwitchStateException {
+                // Read description, if it has been updated
+                if (m.getStatsType() != OFStatsType.DESC) {
+                    log.warn("Expecting Description stats but received stats "
+                            + "type {} from {}. Ignoring ...", m.getStatsType(),
+                            h.channel.getRemoteAddress());
+                    return;
+                }
+                log.info("Received switch description reply from switch at {}",
+                        h.channel.getRemoteAddress());
+                OFDescStatsReply drep = (OFDescStatsReply) m;
+                // Here is where we differentiate between different kinds of switches
+                h.sw = h.controller.getOFSwitchInstance(drep, h.ofVersion);
+                // set switch information
+                h.sw.setOFVersion(h.ofVersion);
+                h.sw.setFeaturesReply(h.featuresReply);
+                h.sw.setPortDescReply(h.portDescReply);
+                h.sw.setConnected(true);
+                h.sw.setChannel(h.channel);
+
+                try {
+                    h.sw.setDebugCounterService(h.controller.getDebugCounter());
+                } catch (CounterException e) {
+                    h.counters.switchCounterRegistrationFailed
+                    .updateCounterNoFlush();
+                    log.warn("Could not register counters for switch {} ",
+                            h.getSwitchInfoString(), e);
+                }
+
+                log.info("Switch {} bound to class {}, description {}",
+                        new Object[] {h.sw, h.sw.getClass(), drep });
+                //Put switch in EQUAL mode until we hear back from the global registry
+                log.debug("Setting new switch {} to EQUAL and sending Role request",
+                        h.sw.getStringId());
+                h.setSwitchRole(Role.EQUAL);
+                try {
+                    boolean supportsRRMsg = h.roleChanger.sendRoleRequest(Role.EQUAL,
+                            RoleRecvStatus.MATCHED_CURRENT_ROLE);
+                    if (!supportsRRMsg) {
+                        log.warn("Switch {} does not support role request messages "
+                                + "of any kind. No role messages were sent. "
+                                + "This controller instance SHOULD become MASTER "
+                                + "from the registry process. ",
+                                h.getSwitchInfoString());
+                    }
+                    h.setState(WAIT_INITIAL_ROLE);
+                    // request control of switch from global registry -
+                    // necessary even if this is the only controller the
+                    // switch is connected to.
+                    h.controller.submitRegistryRequest(h.sw.getId());
+                } catch (IOException e) {
+                    log.error("Exception when sending role request: {} ",
+                            e.getMessage());
+                    // FIXME shouldn't we disconnect?
+                }
+            }
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m) {
+                logErrorDisconnect(h, m);
+            }
+
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                h.pendingPortStatusMsg.add(m);
+            }
+        },
+
+        /**
+         * We are waiting for a role reply message in response to a role request
+         * sent after hearing back from the registry service -- OR -- we are
+         * just waiting to hear back from the registry service in the case that
+         * the switch does not support role messages. If completed successfully,
+         * the controller's role for this switch will be set here.
+         * Before we move to the state corresponding to the role, we allow the
+         * switch specific driver to complete its configuration. This configuration
+         * typically depends on the role the controller is playing for this switch.
+         * And so we set the switch role (for 'this' controller) before we start
+         * the driver-sub-handshake.
+         * Next State: WAIT_SWITCH_DRIVER_SUB_HANDSHAKE
+         */
+        WAIT_INITIAL_ROLE(false) {
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws SwitchStateException {
+                // role changer will ignore the error if it isn't for it
+                RoleRecvStatus rrstatus = h.roleChanger.deliverError(m);
+                if (rrstatus == RoleRecvStatus.OTHER_EXPECTATION) {
+                    logError(h, m);
+                }
+            }
+
+            @Override
+            void processOFExperimenter(OFChannelHandler h, OFExperimenter m)
+                    throws IOException, SwitchStateException {
+                Role role = extractNiciraRoleReply(h, m);
+                // If role == null it means the vendor (experimenter) message
+                // wasn't really a Nicira role reply. We ignore this case.
+                if (role != null) {
+                    RoleReplyInfo rri = new RoleReplyInfo(role, null, m.getXid());
+                    RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(rri);
+                    if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+                        setRoleAndStartDriverHandshake(h, rri.getRole());
+                    } // else do nothing - wait for the correct expected reply
+                } else {
+                    unhandledMessageReceived(h, m);
+                }
+            }
+
+            @Override
+            void processOFRoleReply(OFChannelHandler h, OFRoleReply m)
+                    throws SwitchStateException, IOException {
+                RoleReplyInfo rri = extractOFRoleReply(h, m);
+                RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(rri);
+                if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+                    setRoleAndStartDriverHandshake(h, rri.getRole());
+                } // else do nothing - wait for the correct expected reply
+            }
+
+            @Override
+            void handleUnsentRoleMessage(OFChannelHandler h, Role role,
+                    RoleRecvStatus expectation) throws IOException {
+                // typically this is triggered for a switch where role messages
+                // are not supported - we confirm that the role being set is
+                // master and move to the next state
+                if (expectation == RoleRecvStatus.MATCHED_SET_ROLE) {
+                    if (role == Role.MASTER) {
+                        setRoleAndStartDriverHandshake(h, role);
+                    } else {
+                        log.error("Expected MASTER role from registry for switch "
+                                + "which has no support for role-messages."
+                                + "Received {}. It is possible that this switch "
+                                + "is connected to other controllers, in which "
+                                + "case it should support role messages - not "
+                                + "moving forward.", role);
+                    }
+                } // else do nothing - wait to hear back from registry
+
+            }
+
+            private void setRoleAndStartDriverHandshake(OFChannelHandler h,
+                    Role role) throws IOException {
+                h.setSwitchRole(role);
+                h.sw.startDriverHandshake();
+                if (h.sw.isDriverHandshakeComplete()) {
+                    Role mySwitchRole = h.sw.getRole();
+                    if (mySwitchRole == Role.MASTER) {
+                        log.info("Switch-driver sub-handshake complete. "
+                                + "Activating switch {} with Role: MASTER",
+                                h.getSwitchInfoString());
+                        handlePendingPortStatusMessages(h); //before activation
+                        boolean success = h.controller.addActivatedMasterSwitch(
+                                h.sw.getId(), h.sw);
+                        if (!success) {
+                            disconnectDuplicate(h);
+                            return;
+                        }
+                        h.setState(MASTER);
+                    } else {
+                        log.info("Switch-driver sub-handshake complete. "
+                                + "Activating switch {} with Role: EQUAL",
+                                h.getSwitchInfoString());
+                        handlePendingPortStatusMessages(h); //before activation
+                        boolean success = h.controller.addActivatedEqualSwitch(
+                                h.sw.getId(), h.sw);
+                        if (!success) {
+                            disconnectDuplicate(h);
+                            return;
+                        }
+                        h.setState(EQUAL);
+                    }
+                } else {
+                    h.setState(WAIT_SWITCH_DRIVER_SUB_HANDSHAKE);
+                }
+            }
+
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h, OFStatsReply m)
+                    throws SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                h.pendingPortStatusMsg.add(m);
+
+            }
+        },
+
+        /**
+         * We are waiting for the respective switch driver to complete its
+         * configuration. Notice that we do not consider this to be part of the main
+         * switch-controller handshake. But we do consider it as a step that comes
+         * before we declare the switch as available to the controller.
+         * Next State: depends on the role of this controller for this switch - either
+         * MASTER or EQUAL.
+         */
+        WAIT_SWITCH_DRIVER_SUB_HANDSHAKE(true) {
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException {
+                // will never be called. We override processOFMessage
+            }
+
+            @Override
+            void processOFMessage(OFChannelHandler h, OFMessage m)
+                    throws IOException {
+                if (m.getType() == OFType.ECHO_REQUEST) {
+                    processOFEchoRequest(h, (OFEchoRequest) m);
+                } else {
+                    // FIXME: other message to handle here?
+                    h.sw.processDriverHandshakeMessage(m);
+                    if (h.sw.isDriverHandshakeComplete()) {
+                        // consult the h.sw role and goto that state
+                        Role mySwitchRole = h.sw.getRole();
+                        if (mySwitchRole == Role.MASTER) {
+                            log.info("Switch-driver sub-handshake complete. "
+                                    + "Activating switch {} with Role: MASTER",
+                                    h.getSwitchInfoString());
+                            handlePendingPortStatusMessages(h); //before activation
+                            boolean success = h.controller.addActivatedMasterSwitch(
+                                    h.sw.getId(), h.sw);
+                            if (!success) {
+                                disconnectDuplicate(h);
+                                return;
+                            }
+                            h.setState(MASTER);
+                        } else {
+                            log.info("Switch-driver sub-handshake complete. "
+                                    + "Activating switch {} with Role: EQUAL",
+                                    h.getSwitchInfoString());
+                            handlePendingPortStatusMessages(h); //before activation
+                            boolean success = h.controller.addActivatedEqualSwitch(
+                                    h.sw.getId(), h.sw);
+                            if (!success) {
+                                disconnectDuplicate(h);
+                                return;
+                            }
+                            h.setState(EQUAL);
+                        }
+                    }
+                }
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                h.pendingPortStatusMsg.add(m);
+            }
+        },
+
+
+        /**
+         * This controller is in MASTER role for this switch. We enter this state
+         * after requesting and winning control from the global registry.
+         * The main handshake as well as the switch-driver sub-handshake
+         * is complete at this point.
+         * // XXX S reconsider below
+         * In the (near) future we may deterministically assign controllers to
+         * switches at startup.
+         * We only leave this state if the switch disconnects or
+         * if we send a role request for SLAVE /and/ receive the role reply for
+         * SLAVE.
+         */
+        MASTER(true) {
+            @LogMessageDoc(level = "WARN",
+                    message = "Received permission error from switch {} while"
+                            + "being master. Reasserting master role.",
+                            explanation = "The switch has denied an operation likely "
+                                    + "indicating inconsistent controller roles",
+                                    recommendation = "This situation can occurs transiently during role"
+                                            + " changes. If, however, the condition persists or happens"
+                                            + " frequently this indicates a role inconsistency. "
+                                            + LogMessageDoc.CHECK_CONTROLLER)
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException, SwitchStateException {
+                // first check if the error msg is in response to a role-request message
+                RoleRecvStatus rrstatus = h.roleChanger.deliverError(m);
+                if (rrstatus != RoleRecvStatus.OTHER_EXPECTATION) {
+                    // rolechanger has handled the error message - we are done
+                    return;
+                }
+
+                // if we get here, then the error message is for something else
+                if (m.getErrType() == OFErrorType.BAD_REQUEST &&
+                        ((OFBadRequestErrorMsg) m).getCode() ==
+                        OFBadRequestCode.EPERM) {
+                    // We are the master controller and the switch returned
+                    // a permission error. This is a likely indicator that
+                    // the switch thinks we are slave. Reassert our
+                    // role
+                    // FIXME: this could be really bad during role transitions
+                    // if two controllers are master (even if its only for
+                    // a brief period). We might need to see if these errors
+                    // persist before we reassert
+                    h.counters.epermErrorWhileSwitchIsMaster.updateCounterWithFlush();
+                    log.warn("Received permission error from switch {} while" +
+                            "being master. Reasserting master role.",
+                            h.getSwitchInfoString());
+                    //h.controller.reassertRole(h, Role.MASTER);
+                    // XXX S reassert in role changer or reconsider if all this
+                    // stuff is really needed
+                } else if (m.getErrType() == OFErrorType.FLOW_MOD_FAILED &&
+                        ((OFFlowModFailedErrorMsg) m).getCode() ==
+                        OFFlowModFailedCode.ALL_TABLES_FULL) {
+                    h.sw.setTableFull(true);
+                } else {
+                    logError(h, m);
+                }
+                h.dispatchMessage(m);
+            }
+
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                    OFStatsReply m) {
+                h.sw.deliverStatisticsReply(m);
+            }
+
+            @Override
+            void processOFExperimenter(OFChannelHandler h, OFExperimenter m)
+                    throws IOException, SwitchStateException {
+                Role role = extractNiciraRoleReply(h, m);
+                if (role == null) {
+                    // The message wasn't really a Nicira role reply. We just
+                    // dispatch it to the OFMessage listeners in this case.
+                    h.dispatchMessage(m);
+                    return;
+                }
+
+                RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(
+                        new RoleReplyInfo(role, null, m.getXid()));
+                if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+                    checkAndSetRoleTransition(h, role);
+                }
+            }
+
+            @Override
+            void processOFRoleReply(OFChannelHandler h, OFRoleReply m)
+                    throws SwitchStateException, IOException {
+                RoleReplyInfo rri = extractOFRoleReply(h, m);
+                RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(rri);
+                if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+                    checkAndSetRoleTransition(h, rri.getRole());
+                }
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                handlePortStatusMessage(h, m, true);
+                h.dispatchMessage(m);
+            }
+
+            @Override
+            void processOFPacketIn(OFChannelHandler h, OFPacketIn m)
+                    throws IOException {
+                h.dispatchMessage(m);
+            }
+
+            @Override
+            void processOFFlowRemoved(OFChannelHandler h,
+                    OFFlowRemoved m) throws IOException {
+                h.dispatchMessage(m);
+            }
+
+            @Override
+            void processOFBarrierReply(OFChannelHandler h, OFBarrierReply m)
+                    throws IOException {
+                h.dispatchMessage(m);
+            }
+
+        },
+
+        /**
+         * This controller is in EQUAL role for this switch. We enter this state
+         * after some /other/ controller instance wins mastership-role over this
+         * switch. The EQUAL role can be considered the same as the SLAVE role
+         * if this controller does NOT send commands or packets to the switch.
+         * This should always be true for OF1.0 switches. XXX S need to enforce.
+         *
+         * For OF1.3 switches, choosing this state as EQUAL instead of SLAVE,
+         * gives us the flexibility that if an app wants to send commands/packets
+         * to switches, it can, even thought it is running on a controller instance
+         * that is not in a MASTER role for this switch. Of course, it is the job
+         * of the app to ensure that commands/packets sent by this (EQUAL) controller
+         * instance does not clash/conflict with commands/packets sent by the MASTER
+         * controller for this switch. Neither the controller instances, nor the
+         * switch provides any kind of resolution mechanism should conflicts occur.
+         */
+        EQUAL(true) {
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException, SwitchStateException {
+                // role changer will ignore the error if it isn't for it
+                RoleRecvStatus rrstatus = h.roleChanger.deliverError(m);
+                if (rrstatus == RoleRecvStatus.OTHER_EXPECTATION) {
+                    logError(h, m);
+                    h.dispatchMessage(m);
+                }
+            }
+
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                    OFStatsReply m) {
+                h.sw.deliverStatisticsReply(m);
+            }
+
+            @Override
+            void processOFExperimenter(OFChannelHandler h, OFExperimenter m)
+                    throws IOException, SwitchStateException {
+                Role role = extractNiciraRoleReply(h, m);
+                // If role == null it means the message wasn't really a
+                // Nicira role reply. We ignore it in this state.
+                if (role != null) {
+                    RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(
+                            new RoleReplyInfo(role, null, m.getXid()));
+                    if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+                        checkAndSetRoleTransition(h, role);
+                    }
+                } else {
+                    unhandledMessageReceived(h, m);
+                }
+            }
+
+            @Override
+            void processOFRoleReply(OFChannelHandler h, OFRoleReply m)
+                    throws SwitchStateException, IOException {
+                RoleReplyInfo rri = extractOFRoleReply(h, m);
+                RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(rri);
+                if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+                    checkAndSetRoleTransition(h, rri.getRole());
+                }
+            }
+
+            // XXX S needs more handlers for 1.3 switches in equal role
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                handlePortStatusMessage(h, m, true);
+            }
+
+            @Override
+            @LogMessageDoc(level = "WARN",
+            message = "Received PacketIn from switch {} while "
+                    + "being slave. Reasserting slave role.",
+                    explanation = "The switch has receive a PacketIn despite being "
+                            + "in slave role indicating inconsistent controller roles",
+                            recommendation = "This situation can occurs transiently during role"
+                                    + " changes. If, however, the condition persists or happens"
+                                    + " frequently this indicates a role inconsistency. "
+                                    + LogMessageDoc.CHECK_CONTROLLER)
+            void processOFPacketIn(OFChannelHandler h, OFPacketIn m) throws IOException {
+                // we don't expect packetIn while slave, reassert we are slave
+                h.counters.packetInWhileSwitchIsSlave.updateCounterNoFlush();
+                log.warn("Received PacketIn from switch {} while" +
+                        "being slave. Reasserting slave role.", h.sw);
+                //h.controller.reassertRole(h, Role.SLAVE);
+                // XXX reassert in role changer
+            }
+        };
+
+        private final boolean handshakeComplete;
+        ChannelState(boolean handshakeComplete) {
+            this.handshakeComplete = handshakeComplete;
+        }
+
+        /**
+         * Is this a state in which the handshake has completed?
+         * @return true if the handshake is complete
+         */
+        public boolean isHandshakeComplete() {
+            return handshakeComplete;
+        }
+
+        /**
+         * Get a string specifying the switch connection, state, and
+         * message received. To be used as message for SwitchStateException
+         * or log messages
+         * @param h The channel handler (to get switch information_
+         * @param m The OFMessage that has just been received
+         * @param details A string giving more details about the exact nature
+         * of the problem.
+         * @return
+         */
+        // needs to be protected because enum members are actually subclasses
+        protected String getSwitchStateMessage(OFChannelHandler h,
+                OFMessage m,
+                String details) {
+            return String.format("Switch: [%s], State: [%s], received: [%s]"
+                    + ", details: %s",
+                    h.getSwitchInfoString(),
+                    this.toString(),
+                    m.getType().toString(),
+                    details);
+        }
+
+        /**
+         * We have an OFMessage we didn't expect given the current state and
+         * we want to treat this as an error.
+         * We currently throw an exception that will terminate the connection
+         * However, we could be more forgiving
+         * @param h the channel handler that received the message
+         * @param m the message
+         * @throws SwitchStateException
+         * @throws SwitchStateExeption we always through the execption
+         */
+        // needs to be protected because enum members are acutally subclasses
+        protected void illegalMessageReceived(OFChannelHandler h, OFMessage m)
+                throws SwitchStateException {
+            String msg = getSwitchStateMessage(h, m,
+                    "Switch should never send this message in the current state");
+            throw new SwitchStateException(msg);
+
+        }
+
+        /**
+         * We have an OFMessage we didn't expect given the current state and
+         * we want to ignore the message.
+         * @param h the channel handler the received the message
+         * @param m the message
+         */
+        protected void unhandledMessageReceived(OFChannelHandler h,
+                OFMessage m) {
+            h.counters.unhandledMessage.updateCounterNoFlush();
+            if (log.isDebugEnabled()) {
+                String msg = getSwitchStateMessage(h, m,
+                        "Ignoring unexpected message");
+                log.debug(msg);
+            }
+        }
+
+        /**
+         * Log an OpenFlow error message from a switch.
+         * @param sw The switch that sent the error
+         * @param error The error message
+         */
+        @LogMessageDoc(level = "ERROR",
+                message = "Error {error type} {error code} from {switch} "
+                        + "in state {state}",
+                        explanation = "The switch responded with an unexpected error"
+                                + "to an OpenFlow message from the controller",
+                                recommendation = "This could indicate improper network operation. "
+                                        + "If the problem persists restarting the switch and "
+                                        + "controller may help."
+                )
+        protected void logError(OFChannelHandler h, OFErrorMsg error) {
+            log.error("{} from switch {} in state {}",
+                    new Object[] {
+                    error,
+                    h.getSwitchInfoString(),
+                    this.toString()});
+        }
+
+        /**
+         * Log an OpenFlow error message from a switch and disconnect the
+         * channel.
+         *
+         * @param h the IO channel for this switch.
+         * @param error The error message
+         */
+        protected void logErrorDisconnect(OFChannelHandler h, OFErrorMsg error) {
+            logError(h, error);
+            h.channel.disconnect();
+        }
+
+        /**
+         * log an error message for a duplicate dpid and disconnect this channel.
+         * @param h the IO channel for this switch.
+         */
+        protected void disconnectDuplicate(OFChannelHandler h) {
+            log.error("Duplicated dpid or incompleted cleanup - "
+                    + "disconnecting channel {}", h.getSwitchInfoString());
+            h.duplicateDpidFound = Boolean.TRUE;
+            h.channel.disconnect();
+        }
+
+        /**
+         * Extract the role from an OFVendor message.
+         *
+         * Extract the role from an OFVendor message if the message is a
+         * Nicira role reply. Otherwise return null.
+         *
+         * @param h The channel handler receiving the message
+         * @param vendorMessage The vendor message to parse.
+         * @return The role in the message if the message is a Nicira role
+         * reply, null otherwise.
+         * @throws SwitchStateException If the message is a Nicira role reply
+         * but the numeric role value is unknown.
+         */
+        protected Role extractNiciraRoleReply(OFChannelHandler h,
+                OFExperimenter experimenterMsg) throws SwitchStateException {
+            int vendor = (int) experimenterMsg.getExperimenter();
+            if (vendor != 0x2320) {
+                return null;
+            }
+            OFNiciraControllerRoleReply nrr =
+                    (OFNiciraControllerRoleReply) experimenterMsg;
+
+            Role role = null;
+            OFNiciraControllerRole ncr = nrr.getRole();
+            switch(ncr) {
+            case ROLE_MASTER:
+                role = Role.MASTER;
+                break;
+            case ROLE_OTHER:
+                role = Role.EQUAL;
+                break;
+            case ROLE_SLAVE:
+                role = Role.SLAVE;
+                break;
+            default: //handled below
+            }
+
+            if (role == null) {
+                String msg = String.format("Switch: [%s], State: [%s], "
+                        + "received NX_ROLE_REPLY with invalid role "
+                        + "value %s",
+                        h.getSwitchInfoString(),
+                        this.toString(),
+                        nrr.getRole());
+                throw new SwitchStateException(msg);
+            }
+            return role;
+        }
+
+        /**
+         * Helper class returns role reply information in the format understood
+         * by the controller.
+         */
+        protected static class RoleReplyInfo {
+            private Role role;
+            private U64 genId;
+            private long xid;
+
+            RoleReplyInfo(Role role, U64 genId, long xid) {
+                this.role = role;
+                this.genId = genId;
+                this.xid = xid;
+            }
+            public Role getRole() { return role; }
+            public U64 getGenId() { return genId; }
+            public long getXid() { return xid; }
+            @Override
+            public String toString() {
+                return "[Role:" + role + " GenId:" + genId + " Xid:" + xid + "]";
+            }
+        }
+
+        /**
+         * Extract the role information from an OF1.3 Role Reply Message.
+         * @param h
+         * @param rrmsg
+         * @return RoleReplyInfo object
+         * @throws SwitchStateException
+         */
+        protected RoleReplyInfo extractOFRoleReply(OFChannelHandler h,
+                OFRoleReply rrmsg) throws SwitchStateException {
+            OFControllerRole cr = rrmsg.getRole();
+            Role role = null;
+            switch(cr) {
+            case ROLE_EQUAL:
+                role = Role.EQUAL;
+                break;
+            case ROLE_MASTER:
+                role = Role.MASTER;
+                break;
+            case ROLE_SLAVE:
+                role = Role.SLAVE;
+                break;
+            case ROLE_NOCHANGE: // switch should send current role
+            default:
+                String msg = String.format("Unknown controller role %s "
+                        + "received from switch %s", cr, h.sw);
+                throw new SwitchStateException(msg);
+            }
+
+            return new RoleReplyInfo(role, rrmsg.getGenerationId(), rrmsg.getXid());
+        }
+
+        /**
+         * Handles all pending port status messages before a switch is declared
+         * activated in MASTER or EQUAL role. Note that since this handling
+         * precedes the activation (and therefore notification to IOFSwitchListerners)
+         * the changes to ports will already be visible once the switch is
+         * activated. As a result, no notifications are sent out for these
+         * pending portStatus messages.
+         * @param h
+         * @throws SwitchStateException
+         */
+        protected void handlePendingPortStatusMessages(OFChannelHandler h) {
+            try {
+                handlePendingPortStatusMessages(h, 0);
+            } catch (SwitchStateException e) {
+                log.error(e.getMessage());
+            }
+        }
+
+        private void handlePendingPortStatusMessages(OFChannelHandler h, int index)
+                throws SwitchStateException {
+            if (h.sw == null) {
+                String msg = "State machine error: switch is null. Should never " +
+                        "happen";
+                throw new SwitchStateException(msg);
+            }
+            ArrayList<OFPortStatus> temp  = new ArrayList<OFPortStatus>();
+            for (OFPortStatus ps: h.pendingPortStatusMsg) {
+                temp.add(ps);
+                handlePortStatusMessage(h, ps, false);
+            }
+            temp.clear();
+            // expensive but ok - we don't expect too many port-status messages
+            // note that we cannot use clear(), because of the reasons below
+            h.pendingPortStatusMsg.removeAll(temp);
+            // the iterator above takes a snapshot of the list - so while we were
+            // dealing with the pending port-status messages, we could have received
+            // newer ones. Handle them recursively, but break the recursion after
+            // five steps to avoid an attack.
+            if (!h.pendingPortStatusMsg.isEmpty() && ++index < 5) {
+                handlePendingPortStatusMessages(h, index);
+            }
+        }
+
+        /**
+         * Handle a port status message.
+         *
+         * Handle a port status message by updating the port maps in the
+         * IOFSwitch instance and notifying Controller about the change so
+         * it can dispatch a switch update.
+         *
+         * @param h The OFChannelHhandler that received the message
+         * @param m The PortStatus message we received
+         * @param doNotify if true switch port changed events will be
+         * dispatched
+         * @throws SwitchStateException
+         *
+         */
+        protected void handlePortStatusMessage(OFChannelHandler h, OFPortStatus m,
+                boolean doNotify) throws SwitchStateException {
+            if (h.sw == null) {
+                String msg = getSwitchStateMessage(h, m,
+                        "State machine error: switch is null. Should never " +
+                        "happen");
+                throw new SwitchStateException(msg);
+            }
+
+            Collection<PortChangeEvent> changes = h.sw.processOFPortStatus(m);
+            if (doNotify) {
+                for (PortChangeEvent ev: changes) {
+                    h.controller.notifyPortChanged(h.sw.getId(), ev.port, ev.type);
+                }
+            }
+        }
+
+        /**
+         * Checks if the role received (from the role-reply msg) is different
+         * from the existing role in the IOFSwitch object for this controller.
+         * If so, it transitions the controller to the new role. Note that
+         * the caller should have already verified that the role-reply msg
+         * received was in response to a role-request msg sent out by this
+         * controller after hearing from the registry service.
+         *
+         * @param h the ChannelHandler that received the message
+         * @param role the role in the recieved role reply message
+         */
+        protected void checkAndSetRoleTransition(OFChannelHandler h, Role role) {
+            // we received a role-reply in response to a role message
+            // sent after hearing from the registry service. It is
+            // possible that the role of this controller instance for
+            // this switch has changed:
+            // for 1.0 switch: from MASTER to SLAVE
+            // for 1.3 switch: from MASTER to EQUAL
+            if ((h.sw.getRole() == Role.MASTER && role == Role.SLAVE) ||
+                    (h.sw.getRole() == Role.MASTER && role == Role.EQUAL)) {
+                // the mastership has changed
+                h.sw.setRole(role);
+                h.setState(EQUAL);
+                h.controller.transitionToEqualSwitch(h.sw.getId());
+                return;
+            }
+
+            // or for both 1.0 and 1.3 switches from EQUAL to MASTER.
+            // note that for 1.0, even though we mean SLAVE,
+            // internally we call the role EQUAL.
+            if (h.sw.getRole() == Role.EQUAL && role == Role.MASTER) {
+                // the mastership has changed
+                h.sw.setRole(role);
+                h.setState(MASTER);
+                h.controller.transitionToMasterSwitch(h.sw.getId());
+                return;
+            }
+        }
+
+        /**
+         * Process an OF message received on the channel and
+         * update state accordingly.
+         *
+         * The main "event" of the state machine. Process the received message,
+         * send follow up message if required and update state if required.
+         *
+         * Switches on the message type and calls more specific event handlers
+         * for each individual OF message type. If we receive a message that
+         * is supposed to be sent from a controller to a switch we throw
+         * a SwitchStateExeption.
+         *
+         * The more specific handlers can also throw SwitchStateExceptions
+         *
+         * @param h The OFChannelHandler that received the message
+         * @param m The message we received.
+         * @throws SwitchStateException
+         * @throws IOException
+         */
+        void processOFMessage(OFChannelHandler h, OFMessage m)
+                throws IOException, SwitchStateException {
+            h.roleChanger.checkTimeout();
+            switch(m.getType()) {
+            case HELLO:
+                processOFHello(h, (OFHello) m);
+                break;
+            case BARRIER_REPLY:
+                processOFBarrierReply(h, (OFBarrierReply) m);
+                break;
+            case ECHO_REPLY:
+                processOFEchoReply(h, (OFEchoReply) m);
+                break;
+            case ECHO_REQUEST:
+                processOFEchoRequest(h, (OFEchoRequest) m);
+                break;
+            case ERROR:
+                processOFError(h, (OFErrorMsg) m);
+                break;
+            case FEATURES_REPLY:
+                processOFFeaturesReply(h, (OFFeaturesReply) m);
+                break;
+            case FLOW_REMOVED:
+                processOFFlowRemoved(h, (OFFlowRemoved) m);
+                break;
+            case GET_CONFIG_REPLY:
+                processOFGetConfigReply(h, (OFGetConfigReply) m);
+                break;
+            case PACKET_IN:
+                processOFPacketIn(h, (OFPacketIn) m);
+                break;
+            case PORT_STATUS:
+                processOFPortStatus(h, (OFPortStatus) m);
+                break;
+            case QUEUE_GET_CONFIG_REPLY:
+                processOFQueueGetConfigReply(h, (OFQueueGetConfigReply) m);
+                break;
+            case STATS_REPLY: // multipart_reply in 1.3
+            processOFStatisticsReply(h, (OFStatsReply) m);
+            break;
+            case EXPERIMENTER:
+                processOFExperimenter(h, (OFExperimenter) m);
+                break;
+            case ROLE_REPLY:
+                processOFRoleReply(h, (OFRoleReply) m);
+                break;
+            case GET_ASYNC_REPLY:
+                processOFGetAsyncReply(h, (OFAsyncGetReply) m);
+                break;
+
+                // The following messages are sent to switches. The controller
+                // should never receive them
+            case SET_CONFIG:
+            case GET_CONFIG_REQUEST:
+            case PACKET_OUT:
+            case PORT_MOD:
+            case QUEUE_GET_CONFIG_REQUEST:
+            case BARRIER_REQUEST:
+            case STATS_REQUEST: // multipart request in 1.3
+            case FEATURES_REQUEST:
+            case FLOW_MOD:
+            case GROUP_MOD:
+            case TABLE_MOD:
+            case GET_ASYNC_REQUEST:
+            case SET_ASYNC:
+            case METER_MOD:
+            default:
+                illegalMessageReceived(h, m);
+                break;
+            }
+        }
+
+        /*-----------------------------------------------------------------
+         * Default implementation for message handlers in any state.
+         *
+         * Individual states must override these if they want a behavior
+         * that differs from the default.
+         *
+         * In general, these handlers simply ignore the message and do
+         * nothing.
+         *
+         * There are some exceptions though, since some messages really
+         * are handled the same way in every state (e.g., ECHO_REQUST) or
+         * that are only valid in a single state (e.g., HELLO, GET_CONFIG_REPLY
+         -----------------------------------------------------------------*/
+
+        void processOFHello(OFChannelHandler h, OFHello m)
+                throws IOException, SwitchStateException {
+            // we only expect hello in the WAIT_HELLO state
+            illegalMessageReceived(h, m);
+        }
+
+        void processOFBarrierReply(OFChannelHandler h, OFBarrierReply m)
+                throws IOException {
+            // Silently ignore.
+        }
+
+        void processOFEchoRequest(OFChannelHandler h, OFEchoRequest m)
+                throws IOException {
+            if (h.ofVersion == null) {
+                log.error("No OF version set for {}. Not sending Echo REPLY",
+                        h.channel.getRemoteAddress());
+                return;
+            }
+            OFFactory factory = (h.ofVersion == OFVersion.OF_13) ?
+                    h.controller.getOFMessageFactory13() : h.controller.getOFMessageFactory10();
+            OFEchoReply reply = factory
+                    .buildEchoReply()
+                    .setXid(m.getXid())
+                    .setData(m.getData())
+                    .build();
+            h.channel.write(Collections.singletonList(reply));
+        }
+
+        void processOFEchoReply(OFChannelHandler h, OFEchoReply m)
+                throws IOException {
+            // Do nothing with EchoReplies !!
+        }
+
+        // no default implementation for OFError
+        // every state must override it
+        abstract void processOFError(OFChannelHandler h, OFErrorMsg m)
+                throws IOException, SwitchStateException;
+
+
+        void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                throws IOException, SwitchStateException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFFlowRemoved(OFChannelHandler h, OFFlowRemoved m)
+                throws IOException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFGetConfigReply(OFChannelHandler h, OFGetConfigReply m)
+                throws IOException, SwitchStateException {
+            // we only expect config replies in the WAIT_CONFIG_REPLY state
+            illegalMessageReceived(h, m);
+        }
+
+        void processOFPacketIn(OFChannelHandler h, OFPacketIn m)
+                throws IOException {
+            unhandledMessageReceived(h, m);
+        }
+
+        // no default implementation. Every state needs to handle it.
+        abstract void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                throws IOException, SwitchStateException;
+
+        void processOFQueueGetConfigReply(OFChannelHandler h,
+                OFQueueGetConfigReply m)
+                        throws IOException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFStatisticsReply(OFChannelHandler h, OFStatsReply m)
+                throws IOException, SwitchStateException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFExperimenter(OFChannelHandler h, OFExperimenter m)
+                throws IOException, SwitchStateException {
+            // TODO: it might make sense to parse the vendor message here
+            // into the known vendor messages we support and then call more
+            // specific event handlers
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFRoleReply(OFChannelHandler h, OFRoleReply m)
+                throws SwitchStateException, IOException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFGetAsyncReply(OFChannelHandler h,
+                OFAsyncGetReply m) {
+            unhandledMessageReceived(h, m);
+        }
+
+        void handleUnsentRoleMessage(OFChannelHandler h, Role role,
+                RoleRecvStatus expectation) throws IOException {
+            // do nothing in most states
+        }
+    }
+
+
+
+    //*************************
+    //  Channel handler methods
+    //*************************
+
+    @Override
+    @LogMessageDoc(message = "New switch connection from {ip address}",
+    explanation = "A new switch has connected from the "
+            + "specified IP address")
+    public void channelConnected(ChannelHandlerContext ctx,
+            ChannelStateEvent e) throws Exception {
+        counters.switchConnected.updateCounterWithFlush();
+        channel = e.getChannel();
+        log.info("New switch connection from {}",
+                channel.getRemoteAddress());
+        sendHandshakeHelloMessage();
+        setState(ChannelState.WAIT_HELLO);
+    }
+
+    @Override
+    @LogMessageDoc(message = "Disconnected switch {switch information}",
+    explanation = "The specified switch has disconnected.")
+    public void channelDisconnected(ChannelHandlerContext ctx,
+            ChannelStateEvent e) throws Exception {
+        log.info("Switch disconnected callback for sw:{}. Cleaning up ...",
+                getSwitchInfoString());
+        if (thisdpid != 0) {
+            if (!duplicateDpidFound) {
+                // if the disconnected switch (on this ChannelHandler)
+                // was not one with a duplicate-dpid, it is safe to remove all
+                // state for it at the controller. Notice that if the disconnected
+                // switch was a duplicate-dpid, calling the method below would clear
+                // all state for the original switch (with the same dpid),
+                // which we obviously don't want.
+                controller.removeConnectedSwitch(thisdpid);
+            } else {
+                // A duplicate was disconnected on this ChannelHandler,
+                // this is the same switch reconnecting, but the original state was
+                // not cleaned up - XXX check liveness of original ChannelHandler
+                duplicateDpidFound = Boolean.FALSE;
+            }
+        } else {
+            log.warn("no dpid in channelHandler registered for "
+                    + "disconnected switch {}", getSwitchInfoString());
+        }
+    }
+
+    @Override
+    @LogMessageDocs({
+        @LogMessageDoc(level = "ERROR",
+                message = "Disconnecting switch {switch} due to read timeout",
+                explanation = "The connected switch has failed to send any "
+                        + "messages or respond to echo requests",
+                recommendation = LogMessageDoc.CHECK_SWITCH),
+                        @LogMessageDoc(level = "ERROR",
+                message = "Disconnecting switch {switch}: failed to "
+                        + "complete handshake",
+                explanation = "The switch did not respond correctly "
+                        + "to handshake messages",
+                recommendation = LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level = "ERROR",
+                message = "Disconnecting switch {switch} due to IO Error: {}",
+                explanation = "There was an error communicating with the switch",
+                recommendation = LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level = "ERROR",
+                message = "Disconnecting switch {switch} due to switch "
+                        + "state error: {error}",
+                explanation = "The switch sent an unexpected message",
+                recommendation = LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level = "ERROR",
+                message = "Disconnecting switch {switch} due to "
+                          + "message parse failure",
+                explanation = "Could not parse a message from the switch",
+                recommendation = LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level = "ERROR",
+                message = "Terminating controller due to storage exception",
+                explanation = Controller.ERROR_DATABASE,
+                recommendation = LogMessageDoc.CHECK_CONTROLLER),
+        @LogMessageDoc(level = "ERROR",
+                message = "Could not process message: queue full",
+                explanation = "OpenFlow messages are arriving faster than "
+                            + "the controller can process them.",
+                recommendation = LogMessageDoc.CHECK_CONTROLLER),
+        @LogMessageDoc(level = "ERROR",
+                message = "Error while processing message "
+                        + "from switch {switch} {cause}",
+                explanation = "An error occurred processing the switch message",
+                recommendation = LogMessageDoc.GENERIC_ACTION)
+    })
+    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e)
+            throws Exception {
+        if (e.getCause() instanceof ReadTimeoutException) {
+            // switch timeout
+            log.error("Disconnecting switch {} due to read timeout",
+                    getSwitchInfoString());
+            counters.switchDisconnectReadTimeout.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof HandshakeTimeoutException) {
+            log.error("Disconnecting switch {}: failed to complete handshake",
+                    getSwitchInfoString());
+            counters.switchDisconnectHandshakeTimeout.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof ClosedChannelException) {
+            log.debug("Channel for sw {} already closed", getSwitchInfoString());
+        } else if (e.getCause() instanceof IOException) {
+            log.error("Disconnecting switch {} due to IO Error: {}",
+                    getSwitchInfoString(), e.getCause().getMessage());
+            if (log.isDebugEnabled()) {
+                // still print stack trace if debug is enabled
+                log.debug("StackTrace for previous Exception: ", e.getCause());
+            }
+            counters.switchDisconnectIOError.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof SwitchStateException) {
+            log.error("Disconnecting switch {} due to switch state error: {}",
+                    getSwitchInfoString(), e.getCause().getMessage());
+            if (log.isDebugEnabled()) {
+                // still print stack trace if debug is enabled
+                log.debug("StackTrace for previous Exception: ", e.getCause());
+            }
+            counters.switchDisconnectSwitchStateException.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof OFParseError) {
+            log.error("Disconnecting switch "
+                    + getSwitchInfoString() +
+                    " due to message parse failure",
+                    e.getCause());
+            counters.switchDisconnectParseError.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof RejectedExecutionException) {
+            log.warn("Could not process message: queue full");
+            counters.rejectedExecutionException.updateCounterWithFlush();
+        } else {
+            log.error("Error while processing message from switch "
+                    + getSwitchInfoString()
+                    + "state " + this.state, e.getCause());
+            counters.switchDisconnectOtherException.updateCounterWithFlush();
+            ctx.getChannel().close();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return getSwitchInfoString();
+    }
+
+    @Override
+    public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e)
+            throws Exception {
+        OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+        OFMessage m = factory.buildEchoRequest().build();
+        log.info("Sending Echo Request on idle channel: {}",
+                e.getChannel().getPipeline().getLast().toString());
+        e.getChannel().write(Collections.singletonList(m));
+        // XXX S some problems here -- echo request has no transaction id, and
+        // echo reply is not correlated to the echo request.
+    }
+
+    @Override
+    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e)
+            throws Exception {
+        if (e.getMessage() instanceof List) {
+            @SuppressWarnings("unchecked")
+            List<OFMessage> msglist = (List<OFMessage>) e.getMessage();
+
+
+            for (OFMessage ofm : msglist) {
+                counters.messageReceived.updateCounterNoFlush();
+                // Do the actual packet processing
+                state.processOFMessage(this, ofm);
+            }
+        } else {
+            counters.messageReceived.updateCounterNoFlush();
+            state.processOFMessage(this, (OFMessage) e.getMessage());
+        }
+    }
+
+
+
+    //*************************
+    //  Channel utility methods
+    //*************************
+
+    /**
+     * Is this a state in which the handshake has completed?
+     * @return true if the handshake is complete
+     */
+    public boolean isHandshakeComplete() {
+        return this.state.isHandshakeComplete();
+    }
+
+    private void dispatchMessage(OFMessage m) throws IOException {
+        sw.handleMessage(m);
+    }
+
+    /**
+     * Return a string describing this switch based on the already available
+     * information (DPID and/or remote socket).
+     * @return
+     */
+    private String getSwitchInfoString() {
+        if (sw != null) {
+            return sw.toString();
+        }
+        String channelString;
+        if (channel == null || channel.getRemoteAddress() == null) {
+            channelString = "?";
+        } else {
+            channelString = channel.getRemoteAddress().toString();
+        }
+        String dpidString;
+        if (featuresReply == null) {
+            dpidString = "?";
+        } else {
+            dpidString = featuresReply.getDatapathId().toString();
+        }
+        return String.format("[%s DPID[%s]]", channelString, dpidString);
+    }
+
+    /**
+     * Update the channels state. Only called from the state machine.
+     * TODO: enforce restricted state transitions
+     * @param state
+     */
+    private void setState(ChannelState state) {
+        this.state = state;
+    }
+
+    /**
+     * Send hello message to the switch using the handshake transactions ids.
+     * @throws IOException
+     */
+    private void sendHandshakeHelloMessage() throws IOException {
+        // The OF protocol requires us to start things off by sending the highest
+        // version of the protocol supported.
+
+        // bitmap represents OF1.0 (ofp_version=0x01) and OF1.3 (ofp_version=0x04)
+        // see Sec. 7.5.1 of the OF1.3.4 spec
+        U32 bitmap = U32.ofRaw(0x00000012);
+        OFHelloElem hem = factory13.buildHelloElemVersionbitmap()
+                .setBitmaps(Collections.singletonList(bitmap))
+                .build();
+        OFMessage.Builder mb = factory13.buildHello()
+                .setXid(this.handshakeTransactionIds--)
+                .setElements(Collections.singletonList(hem));
+        log.info("Sending OF_13 Hello to {}", channel.getRemoteAddress());
+        channel.write(Collections.singletonList(mb.build()));
+    }
+
+    /**
+     * Send featuresRequest msg to the switch using the handshake transactions ids.
+     * @throws IOException
+     */
+    private void sendHandshakeFeaturesRequestMessage() throws IOException {
+        OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+        OFMessage m = factory.buildFeaturesRequest()
+                .setXid(this.handshakeTransactionIds--)
+                .build();
+        channel.write(Collections.singletonList(m));
+    }
+
+    private void setSwitchRole(Role role) {
+        sw.setRole(role);
+    }
+
+    /**
+     * Send the configuration requests to tell the switch we want full
+     * packets.
+     * @throws IOException
+     */
+    private void sendHandshakeSetConfig() throws IOException {
+        OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+        //log.debug("Sending CONFIG_REQUEST to {}", channel.getRemoteAddress());
+        List<OFMessage> msglist = new ArrayList<OFMessage>(3);
+
+        // Ensure we receive the full packet via PacketIn
+        // FIXME: We don't set the reassembly flags.
+        OFSetConfig sc = factory
+                .buildSetConfig()
+                .setMissSendLen((short) 0xffff)
+                .setXid(this.handshakeTransactionIds--)
+                .build();
+        msglist.add(sc);
+
+        // Barrier
+        OFBarrierRequest br = factory
+                .buildBarrierRequest()
+                .setXid(this.handshakeTransactionIds--)
+                .build();
+        msglist.add(br);
+
+        // Verify (need barrier?)
+        OFGetConfigRequest gcr = factory
+                .buildGetConfigRequest()
+                .setXid(this.handshakeTransactionIds--)
+                .build();
+        msglist.add(gcr);
+        channel.write(msglist);
+    }
+
+    /**
+     * send a description state request.
+     * @throws IOException
+     */
+    private void sendHandshakeDescriptionStatsRequest() throws IOException {
+        // Get Description to set switch-specific flags
+        OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+        OFDescStatsRequest dreq = factory
+                .buildDescStatsRequest()
+                .setXid(handshakeTransactionIds--)
+                .build();
+        channel.write(Collections.singletonList(dreq));
+    }
+
+    private void sendHandshakeOFPortDescRequest() throws IOException {
+        // Get port description for 1.3 switch
+        OFPortDescStatsRequest preq = factory13
+                .buildPortDescStatsRequest()
+                .setXid(handshakeTransactionIds--)
+                .build();
+        channel.write(Collections.singletonList(preq));
+    }
+
+    ChannelState getStateForTesting() {
+        return state;
+    }
+
+    void useRoleChangerWithOtherTimeoutForTesting(long roleTimeoutMs) {
+        roleChanger = new RoleChanger(roleTimeoutMs);
+    }
+
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFMessageDecoder.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFMessageDecoder.java
new file mode 100644
index 0000000..ff6fbf5
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFMessageDecoder.java
@@ -0,0 +1,56 @@
+/**
+ *    Copyright 2011, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.internal;
+
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.handler.codec.frame.FrameDecoder;
+import org.projectfloodlight.openflow.protocol.OFFactories;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFMessageReader;
+
+/**
+ * Decode an openflow message from a Channel, for use in a netty pipeline.
+ */
+public class OFMessageDecoder extends FrameDecoder {
+
+    @Override
+    protected Object decode(ChannelHandlerContext ctx, Channel channel,
+            ChannelBuffer buffer) throws Exception {
+        if (!channel.isConnected()) {
+            // In testing, I see decode being called AFTER decode last.
+            // This check avoids that from reading corrupted frames
+            return null;
+        }
+
+        // Note that a single call to decode results in reading a single
+        // OFMessage from the channel buffer, which is passed on to, and processed
+        // by, the controller (in OFChannelHandler).
+        // This is different from earlier behavior (with the original openflowj),
+        // where we parsed all the messages in the buffer, before passing on
+        // a list of the parsed messages to the controller.
+        // The performance *may or may not* not be as good as before.
+        OFMessageReader<OFMessage> reader = OFFactories.getGenericReader();
+        OFMessage message = reader.readFrom(buffer);
+
+        return message;
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFMessageEncoder.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFMessageEncoder.java
new file mode 100644
index 0000000..86933fc
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OFMessageEncoder.java
@@ -0,0 +1,59 @@
+/**
+ *    Copyright 2011, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.internal;
+
+import java.util.List;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.handler.codec.oneone.OneToOneEncoder;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+
+
+/**
+ * Encode an openflow message for output into a ChannelBuffer, for use in a
+ * netty pipeline.
+ */
+public class OFMessageEncoder extends OneToOneEncoder {
+
+    @Override
+    protected Object encode(ChannelHandlerContext ctx, Channel channel,
+                            Object msg) throws Exception {
+        if (!(msg instanceof List)) {
+            return msg;
+        }
+
+        @SuppressWarnings("unchecked")
+        List<OFMessage> msglist = (List<OFMessage>) msg;
+        /* XXX S can't get length of OFMessage in loxigen's openflowj??
+        int size = 0;
+        for (OFMessage ofm : msglist) {
+            size += ofm.getLengthU();
+        }*/
+
+        ChannelBuffer buf = ChannelBuffers.dynamicBuffer();
+
+        for (OFMessage ofm : msglist) {
+            ofm.writeTo(buf);
+        }
+        return buf;
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OpenflowPipelineFactory.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OpenflowPipelineFactory.java
new file mode 100644
index 0000000..30f0287
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/OpenflowPipelineFactory.java
@@ -0,0 +1,78 @@
+/**
+*    Copyright 2011, Big Switch Networks, Inc.
+*    Originally created by David Erickson, Stanford University
+*
+*    Licensed under the Apache License, Version 2.0 (the "License"); you may
+*    not use this file except in compliance with the License. You may obtain
+*    a copy of the License at
+*
+*         http://www.apache.org/licenses/LICENSE-2.0
+*
+*    Unless required by applicable law or agreed to in writing, software
+*    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+*    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+*    License for the specific language governing permissions and limitations
+*    under the License.
+**/
+
+package net.onrc.onos.of.ctl.internal;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.handler.execution.ExecutionHandler;
+import org.jboss.netty.handler.timeout.IdleStateHandler;
+import org.jboss.netty.handler.timeout.ReadTimeoutHandler;
+import org.jboss.netty.util.ExternalResourceReleasable;
+import org.jboss.netty.util.HashedWheelTimer;
+import org.jboss.netty.util.Timer;
+
+/**
+ * Creates a ChannelPipeline for a server-side openflow channel.
+ */
+public class OpenflowPipelineFactory
+    implements ChannelPipelineFactory, ExternalResourceReleasable {
+
+    protected Controller controller;
+    protected ThreadPoolExecutor pipelineExecutor;
+    protected Timer timer;
+    protected IdleStateHandler idleHandler;
+    protected ReadTimeoutHandler readTimeoutHandler;
+
+    public OpenflowPipelineFactory(Controller controller,
+                                   ThreadPoolExecutor pipelineExecutor) {
+        super();
+        this.controller = controller;
+        this.pipelineExecutor = pipelineExecutor;
+        this.timer = new HashedWheelTimer();
+        this.idleHandler = new IdleStateHandler(timer, 20, 25, 0);
+        this.readTimeoutHandler = new ReadTimeoutHandler(timer, 30);
+    }
+
+    @Override
+    public ChannelPipeline getPipeline() throws Exception {
+        OFChannelHandler handler = new OFChannelHandler(controller);
+
+        ChannelPipeline pipeline = Channels.pipeline();
+        pipeline.addLast("ofmessagedecoder", new OFMessageDecoder());
+        pipeline.addLast("ofmessageencoder", new OFMessageEncoder());
+        pipeline.addLast("idle", idleHandler);
+        pipeline.addLast("timeout", readTimeoutHandler);
+        // XXX S ONOS: was 15 increased it to fix Issue #296
+        pipeline.addLast("handshaketimeout",
+                         new HandshakeTimeoutHandler(handler, timer, 60));
+        if (pipelineExecutor != null) {
+            pipeline.addLast("pipelineExecutor",
+                             new ExecutionHandler(pipelineExecutor));
+        }
+        pipeline.addLast("handler", handler);
+        return pipeline;
+    }
+
+    @Override
+    public void releaseExternalResources() {
+        timer.stop();
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeAlreadyStarted.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeAlreadyStarted.java
new file mode 100644
index 0000000..92c673c
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeAlreadyStarted.java
@@ -0,0 +1,14 @@
+package net.onrc.onos.of.ctl.internal;
+
+/**
+ * Thrown when IOFSwitch.startDriverHandshake() is called more than once.
+ *
+ */
+public class SwitchDriverSubHandshakeAlreadyStarted extends
+    SwitchDriverSubHandshakeException {
+    private static final long serialVersionUID = -5491845708752443501L;
+
+    public SwitchDriverSubHandshakeAlreadyStarted() {
+        super();
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeCompleted.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeCompleted.java
new file mode 100644
index 0000000..1600854
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeCompleted.java
@@ -0,0 +1,19 @@
+package net.onrc.onos.of.ctl.internal;
+
+import org.projectfloodlight.openflow.protocol.OFMessage;
+
+
+/**
+ * Indicates that a message was passed to a switch driver's subhandshake
+ * handling code but the driver has already completed the sub-handshake.
+ *
+ */
+public class SwitchDriverSubHandshakeCompleted
+        extends SwitchDriverSubHandshakeException {
+    private static final long serialVersionUID = -8817822245846375995L;
+
+    public SwitchDriverSubHandshakeCompleted(OFMessage m) {
+        super("Sub-Handshake is already complete but received message "
+              + m.getType());
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeException.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeException.java
new file mode 100644
index 0000000..c7d68f3
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeException.java
@@ -0,0 +1,26 @@
+package net.onrc.onos.of.ctl.internal;
+
+/**
+ * Base class for exception thrown by switch driver sub-handshake processing.
+ *
+ */
+public class SwitchDriverSubHandshakeException extends RuntimeException {
+    private static final long serialVersionUID = -6257836781419604438L;
+
+    protected SwitchDriverSubHandshakeException() {
+        super();
+    }
+
+    protected SwitchDriverSubHandshakeException(String arg0, Throwable arg1) {
+        super(arg0, arg1);
+    }
+
+    protected SwitchDriverSubHandshakeException(String arg0) {
+        super(arg0);
+    }
+
+    protected SwitchDriverSubHandshakeException(Throwable arg0) {
+        super(arg0);
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeNotStarted.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeNotStarted.java
new file mode 100644
index 0000000..d568cc6
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeNotStarted.java
@@ -0,0 +1,15 @@
+package net.onrc.onos.of.ctl.internal;
+
+/**
+ * Thrown when a switch driver's sub-handshake has not been started but an
+ * operation requiring the sub-handshake has been attempted.
+ *
+ */
+public class SwitchDriverSubHandshakeNotStarted extends
+    SwitchDriverSubHandshakeException {
+    private static final long serialVersionUID = -5491845708752443501L;
+
+    public SwitchDriverSubHandshakeNotStarted() {
+        super();
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeStateException.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeStateException.java
new file mode 100644
index 0000000..6091a86
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchDriverSubHandshakeStateException.java
@@ -0,0 +1,15 @@
+package net.onrc.onos.of.ctl.internal;
+
+/**
+ * Thrown when a switch driver's sub-handshake state-machine receives an
+ * unexpected OFMessage and/or is in an invald state.
+ *
+ */
+public class SwitchDriverSubHandshakeStateException extends
+    SwitchDriverSubHandshakeException {
+    private static final long serialVersionUID = -8249926069195147051L;
+
+    public SwitchDriverSubHandshakeStateException(String msg) {
+        super(msg);
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchStateException.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchStateException.java
new file mode 100644
index 0000000..e51b60d
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/internal/SwitchStateException.java
@@ -0,0 +1,50 @@
+/**
+ *    Copyright 2011, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.internal;
+
+/**
+ * This exception indicates an error or unexpected message during
+ * message handling. E.g., if an OFMessage is received that is illegal or
+ * unexpected given the current handshake state.
+ *
+ * We don't allow wrapping other exception in a switch state exception. We
+ * only log the SwitchStateExceptions message so the causing exceptions
+ * stack trace is generally not available.
+ *
+ */
+public class SwitchStateException extends Exception {
+
+    private static final long serialVersionUID = 9153954512470002631L;
+
+    public SwitchStateException() {
+        super();
+    }
+
+    public SwitchStateException(String arg0, Throwable arg1) {
+        super(arg0, arg1);
+    }
+
+    public SwitchStateException(String arg0) {
+        super(arg0);
+    }
+
+    public SwitchStateException(Throwable arg0) {
+        super(arg0);
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/ControllerRegistryEntry.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/ControllerRegistryEntry.java
new file mode 100644
index 0000000..2472f64
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/ControllerRegistryEntry.java
@@ -0,0 +1,66 @@
+package net.onrc.onos.of.ctl.registry;
+
+
+
+public class ControllerRegistryEntry implements Comparable<ControllerRegistryEntry> {
+    //
+    // TODO: Refactor the implementation and decide whether controllerId
+    // is needed. If "yes", we might need to consider it inside the
+    // compareTo(), equals() and hashCode() implementations.
+    //
+    private final String controllerId;
+    private final int sequenceNumber;
+
+    public ControllerRegistryEntry(String controllerId, int sequenceNumber) {
+        this.controllerId = controllerId;
+        this.sequenceNumber = sequenceNumber;
+    }
+
+    public String getControllerId() {
+        return controllerId;
+    }
+
+    /**
+     * Compares this object with the specified object for order.
+     * NOTE: the test is based on ControllerRegistryEntry sequence numbers,
+     * and doesn't include the controllerId.
+     *
+     * @param o the object to be compared.
+     * @return a negative integer, zero, or a positive integer as this object
+     * is less than, equal to, or greater than the specified object.
+     */
+    @Override
+    public int compareTo(ControllerRegistryEntry o) {
+        return this.sequenceNumber - o.sequenceNumber;
+    }
+
+    /**
+     * Test whether some other object is "equal to" this one.
+     * NOTE: the test is based on ControllerRegistryEntry sequence numbers,
+     * and doesn't include the controllerId.
+     *
+     * @param obj the reference object with which to compare.
+     * @return true if this object is the same as the obj argument; false
+     * otherwise.
+     */
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof ControllerRegistryEntry) {
+            ControllerRegistryEntry other = (ControllerRegistryEntry) obj;
+            return this.sequenceNumber == other.sequenceNumber;
+        }
+        return false;
+    }
+
+    /**
+     * Get the hash code for the object.
+     * NOTE: the computation is based on ControllerRegistryEntry sequence
+     * numbers, and doesn't include the controller ID.
+     *
+     * @return a hash code value for this object.
+     */
+    @Override
+    public int hashCode() {
+        return Integer.valueOf(this.sequenceNumber).hashCode();
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/IControllerRegistry.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/IControllerRegistry.java
new file mode 100644
index 0000000..0b67338
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/IControllerRegistry.java
@@ -0,0 +1,155 @@
+package net.onrc.onos.of.ctl.registry;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import net.onrc.onos.of.ctl.util.InstanceId;
+
+/**
+ * A registry service that allows ONOS to register controllers and switches in a
+ * way that is global to the entire ONOS cluster. The registry is the arbiter
+ * for allowing controllers to control switches.
+ * <p/>
+ * The OVS/OF1.{2,3} fault tolerance model is a switch connects to multiple
+ * controllers, and the controllers send role requests to tell the switch their
+ * role in controlling the switch.
+ * <p/>
+ * The ONOS fault tolerance model allows only a single controller to have
+ * control of a switch (MASTER role) at once. Controllers therefore need a
+ * mechanism that enables them to decide who should control a each switch. The
+ * registry service provides this mechanism.
+ */
+public interface IControllerRegistry {
+
+    /**
+     * Callback interface for control change events.
+     */
+    public interface ControlChangeCallback {
+        /**
+         * Called whenever the control changes from the point of view of the
+         * registry. The callee can check whether they have control or not using
+         * the hasControl parameter.
+         *
+         * @param dpid The switch that control has changed for
+         * @param hasControl Whether the listener now has control or not
+         */
+        void controlChanged(long dpid, boolean hasControl);
+    }
+
+    /**
+     * Request for control of a switch. This method does not block. When control
+     * for a switch changes, the controlChanged method on the callback object
+     * will be called. This happens any time the control changes while the
+     * request is still active (until releaseControl is called)
+     *
+     * @param dpid Switch to request control for
+     * @param cb Callback that will be used to notify caller of control changes
+     * @throws RegistryException Errors contacting the registry service
+     */
+    public void requestControl(long dpid, ControlChangeCallback cb)
+            throws RegistryException;
+
+    /**
+     * Stop trying to take control of a switch. This removes the entry for this
+     * controller requesting this switch in the registry. If the controller had
+     * control when this is called, another controller will now gain control of
+     * the switch. This call doesn't block.
+     *
+     * @param dpid Switch to release control of
+     */
+    public void releaseControl(long dpid);
+
+    /**
+     * Check whether the controller has control of the switch This call doesn't
+     * block.
+     *
+     * @param dpid Switch to check control of
+     * @return true if controller has control of the switch.
+     */
+    public boolean hasControl(long dpid);
+
+    /**
+     * Check whether this instance is the leader for the cluster. This call
+     * doesn't block.
+     *
+     * @return true if the instance is the leader for the cluster, otherwise
+     *         false.
+     */
+    public boolean isClusterLeader();
+
+    /**
+     * Gets the unique ID used to identify this ONOS instance in the cluster.
+     *
+     * @return Instance ID.
+     */
+    public InstanceId getOnosInstanceId();
+
+    /**
+     * Register a controller to the ONOS cluster. Must be called before the
+     * registry can be used to take control of any switches.
+     *
+     * @param controllerId A unique string ID identifying this controller in the
+     *        cluster
+     * @throws RegistryException for errors connecting to registry service,
+     *         controllerId already registered
+     */
+    public void registerController(String controllerId)
+            throws RegistryException;
+
+    /**
+     * Get all controllers in the cluster.
+     *
+     * @return Collection of controller IDs
+     * @throws RegistryException on error
+     */
+    public Collection<String> getAllControllers() throws RegistryException;
+
+    /**
+     * Get all switches in the cluster, along with which controller is in
+     * control of them (if any) and any other controllers that have requested
+     * control.
+     *
+     * @return Map of all switches.
+     */
+    public Map<String, List<ControllerRegistryEntry>> getAllSwitches();
+
+    /**
+     * Get the controller that has control of a given switch.
+     *
+     * @param dpid Switch to find controller for
+     * @return controller ID
+     * @throws RegistryException Errors contacting registry service
+     */
+    public String getControllerForSwitch(long dpid) throws RegistryException;
+
+    /**
+     * Get all switches controlled by a given controller.
+     *
+     * @param controllerId ID of the controller
+     * @return Collection of dpids
+     */
+    public Collection<Long> getSwitchesControlledByController(String controllerId);
+
+    /**
+     * Get a unique Id Block.
+     *
+     * @return Id Block.
+     */
+    public IdBlock allocateUniqueIdBlock();
+
+    /**
+     * Get next unique id and retrieve a new range of ids if needed.
+     *
+     * @param range range to use for the identifier
+     * @return Id Block.
+     */
+    public IdBlock allocateUniqueIdBlock(long range);
+
+    /**
+     * Get a globally unique ID.
+     *
+     * @return a globally unique ID.
+     */
+    public long getNextUniqueId();
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/IdBlock.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/IdBlock.java
new file mode 100644
index 0000000..45d3c83
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/IdBlock.java
@@ -0,0 +1,32 @@
+package net.onrc.onos.of.ctl.registry;
+
+public class IdBlock {
+    private final long start;
+    private final long end;
+    private final long size;
+
+    public IdBlock(long start, long end, long size) {
+        this.start = start;
+        this.end = end;
+        this.size = size;
+    }
+
+    public long getStart() {
+        return start;
+    }
+
+    public long getEnd() {
+        return end;
+    }
+
+    public long getSize() {
+        return size;
+    }
+
+    @Override
+    public String toString() {
+        return "IdBlock [start=" + start + ", end=" + end + ", size=" + size
+                + "]";
+    }
+}
+
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/RegistryException.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/RegistryException.java
new file mode 100644
index 0000000..06f5932
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/registry/RegistryException.java
@@ -0,0 +1,15 @@
+package net.onrc.onos.of.ctl.registry;
+
+public class RegistryException extends Exception {
+
+    private static final long serialVersionUID = -8276300722010217913L;
+
+    public RegistryException(String message) {
+        super(message);
+    }
+
+    public RegistryException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/Dpid.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/Dpid.java
new file mode 100644
index 0000000..5544354
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/Dpid.java
@@ -0,0 +1,74 @@
+package net.onrc.onos.of.ctl.util;
+
+import org.projectfloodlight.openflow.util.HexString;
+
+/**
+ * The class representing a network switch DPID.
+ * This class is immutable.
+ */
+public final class Dpid {
+    private static final long UNKNOWN = 0;
+    private final long value;
+
+    /**
+     * Default constructor.
+     */
+    public Dpid() {
+        this.value = Dpid.UNKNOWN;
+    }
+
+    /**
+     * Constructor from a long value.
+     *
+     * @param value the value to use.
+     */
+    public Dpid(long value) {
+        this.value = value;
+    }
+
+    /**
+     * Constructor from a string.
+     *
+     * @param value the value to use.
+     */
+    public Dpid(String value) {
+        this.value = HexString.toLong(value);
+    }
+
+    /**
+     * Get the value of the DPID.
+     *
+     * @return the value of the DPID.
+     */
+    public long value() {
+        return value;
+    }
+
+    /**
+     * Convert the DPID value to a ':' separated hexadecimal string.
+     *
+     * @return the DPID value as a ':' separated hexadecimal string.
+     */
+    @Override
+    public String toString() {
+        return HexString.toHexString(this.value);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (!(other instanceof Dpid)) {
+            return false;
+        }
+
+        Dpid otherDpid = (Dpid) other;
+
+        return value == otherDpid.value;
+    }
+
+    @Override
+    public int hashCode() {
+        int hash = 17;
+        hash += 31 * hash + (int) (value ^ value >>> 32);
+        return hash;
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/DummySwitchForTesting.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/DummySwitchForTesting.java
new file mode 100644
index 0000000..a8eabce
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/DummySwitchForTesting.java
@@ -0,0 +1,360 @@
+package net.onrc.onos.of.ctl.util;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Future;
+
+import org.jboss.netty.channel.Channel;
+import org.projectfloodlight.openflow.protocol.OFActionType;
+import org.projectfloodlight.openflow.protocol.OFCapabilities;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFFeaturesReply;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFPortStatus;
+import org.projectfloodlight.openflow.protocol.OFStatsReply;
+import org.projectfloodlight.openflow.protocol.OFStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.types.DatapathId;
+import org.projectfloodlight.openflow.types.U64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.onrc.onos.of.ctl.IOFSwitch;
+import net.onrc.onos.of.ctl.Role;
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounterService;
+import net.onrc.onos.of.ctl.debugcounter.IDebugCounterService.CounterException;
+
+public class DummySwitchForTesting implements IOFSwitch {
+
+    protected static final Logger log = LoggerFactory.getLogger(DummySwitchForTesting.class);
+
+    private Channel channel;
+    private boolean connected = false;
+    private OFVersion ofv = OFVersion.OF_10;
+
+    private Collection<OFPortDesc> ports;
+
+    private DatapathId datapathId;
+
+    private Set<OFCapabilities> capabilities;
+
+    private int buffers;
+
+    private byte tables;
+
+    private String stringId;
+
+    private Role role;
+
+    @Override
+    public void disconnectSwitch() {
+        this.channel.close();
+    }
+
+    @Override
+    public void write(OFMessage m) throws IOException {
+        this.channel.write(m);
+
+    }
+
+    @Override
+    public void write(List<OFMessage> msglist) throws IOException {
+        for (OFMessage m : msglist) {
+            this.channel.write(m);
+        }
+
+    }
+
+    @Override
+    public Date getConnectedSince() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public int getNextTransactionId() {
+        return 0;
+    }
+
+    @Override
+    public boolean isConnected() {
+        return this.connected;
+    }
+
+    @Override
+    public void setConnected(boolean connected) {
+        this.connected  = connected;
+
+    }
+
+    @Override
+    public void flush() {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void setChannel(Channel channel) {
+        this.channel = channel;
+
+    }
+
+    @Override
+    public long getId() {
+        if (this.stringId == null) {
+            throw new RuntimeException("Features reply has not yet been set");
+        }
+        return this.datapathId.getLong();
+    }
+
+    @Override
+    public String getStringId() {
+        // TODO Auto-generated method stub
+        return "DummySwitch";
+    }
+
+    @Override
+    public int getNumBuffers() {
+        // TODO Auto-generated method stub
+        return 0;
+    }
+
+    @Override
+    public Set<OFCapabilities> getCapabilities() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public byte getNumTables() {
+        // TODO Auto-generated method stub
+        return 0;
+    }
+
+    @Override
+    public OFDescStatsReply getSwitchDescription() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public void cancelFeaturesReply(int transactionId) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public Set<OFActionType> getActions() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public void setOFVersion(OFVersion version) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public OFVersion getOFVersion() {
+        return this.ofv;
+    }
+
+    @Override
+    public Collection<OFPortDesc> getEnabledPorts() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Collection<Integer> getEnabledPortNumbers() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public OFPortDesc getPort(int portNumber) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public OFPortDesc getPort(String portName) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public OrderedCollection<PortChangeEvent> processOFPortStatus(
+            OFPortStatus ps) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Collection<OFPortDesc> getPorts() {
+        return ports;
+    }
+
+    @Override
+    public boolean portEnabled(int portName) {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public OrderedCollection<PortChangeEvent> setPorts(
+            Collection<OFPortDesc> p) {
+        this.ports = p;
+        return null;
+    }
+
+    @Override
+    public Map<Object, Object> getAttributes() {
+        return null;
+    }
+
+    @Override
+    public boolean hasAttribute(String name) {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public Object getAttribute(String name) {
+        return Boolean.FALSE;
+    }
+
+    @Override
+    public void setAttribute(String name, Object value) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public Object removeAttribute(String name) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public void deliverStatisticsReply(OFMessage reply) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void cancelStatisticsReply(int transactionId) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void cancelAllStatisticsReplies() {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public Future<List<OFStatsReply>> getStatistics(OFStatsRequest<?> request)
+            throws IOException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public void clearAllFlowMods() {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public Role getRole() {
+        return this.role;
+    }
+
+    @Override
+    public void setRole(Role role) {
+        this.role = role;
+    }
+
+    @Override
+    public U64 getNextGenerationId() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public void setDebugCounterService(IDebugCounterService debugCounter)
+            throws CounterException {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void startDriverHandshake() throws IOException {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public boolean isDriverHandshakeComplete() {
+        return true;
+    }
+
+    @Override
+    public void processDriverHandshakeMessage(OFMessage m) {
+
+    }
+
+    @Override
+    public void setTableFull(boolean isFull) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void setFeaturesReply(OFFeaturesReply featuresReply) {
+        if (featuresReply == null) {
+            log.error("Error setting featuresReply for switch: {}", getStringId());
+            return;
+        }
+        this.datapathId = featuresReply.getDatapathId();
+        this.capabilities = featuresReply.getCapabilities();
+        this.buffers = (int) featuresReply.getNBuffers();
+        this.tables = (byte) featuresReply.getNTables();
+        this.stringId = this.datapathId.toString();
+
+    }
+
+    @Override
+    public void setPortDescReply(OFPortDescStatsReply portDescReply) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void handleMessage(OFMessage m) {
+        log.info("Got packet {} but I am dumb so I don't know what to do.", m);
+    }
+
+    @Override
+    public boolean portEnabled(String portName) {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public OrderedCollection<PortChangeEvent> comparePorts(
+            Collection<OFPortDesc> p) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/EnumBitmaps.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/EnumBitmaps.java
new file mode 100644
index 0000000..fe6ccc0
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/EnumBitmaps.java
@@ -0,0 +1,149 @@
+package net.onrc.onos.of.ctl.util;
+
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * A utility class to convert between integer based bitmaps for (OpenFlow)
+ * flags and Enum and EnumSet based representations.
+ *
+ * The enum used to represent individual flags needs to implement the
+ * BitmapableEnum interface.
+ *
+ * Example:
+ * {@code
+ *   int bitmap = 0x11; // OFPPC_PORT_DOWN | OFPPC_NO_STP
+ *   EnumSet<OFPortConfig> s = toEnumSet(OFPortConfig.class, bitmap);
+ *   // s will contain OFPPC_PORT_DOWN and OFPPC_NO_STP
+ * }
+ *
+ * {@code
+ *    EnumSet<OFPortConfig> s = EnumSet.of(OFPPC_NO_STP, OFPPC_PORT_DOWN);
+ *    int bitmap = toBitmap(s); // returns 0x11
+ * }
+ *
+ */
+public final class EnumBitmaps {
+
+
+    private EnumBitmaps() { }
+
+    /**
+     * Enums used to represent individual flags needs to implement this
+     * interface.
+     */
+    public interface BitmapableEnum {
+        /** Return the value in the bitmap that the enum constant represents.
+         * The returned value must have only a single bit set. E.g.,1 << 3
+         */
+        int getValue();
+    }
+
+
+    /**
+     * Convert an integer bitmap to an EnumSet.
+     *
+     * See class description for example
+     * @param type The Enum class to use. Must implement BitmapableEnum
+     * @param bitmap The integer bitmap
+     * @return A newly allocated EnumSet representing the bits set in the
+     * bitmap
+     * @throws NullPointerException if type is null
+     * @throws IllegalArgumentException if any enum constant from type has
+     * more than one bit set.
+     * @throws IllegalArgumentException if the bitmap has any bits set not
+     * represented by an enum constant.
+     */
+    public static <E extends Enum<E> & BitmapableEnum>
+            EnumSet<E> toEnumSet(Class<E> type, int bitmap) {
+        if (type == null) {
+            throw new NullPointerException("Given enum type must not be null");
+        }
+        EnumSet<E> s = EnumSet.noneOf(type);
+        // allSetBitmap will eventually have all valid bits for the given
+        // type set.
+        int allSetBitmap = 0;
+        for (E element: type.getEnumConstants()) {
+            if (Integer.bitCount(element.getValue()) != 1) {
+                String msg = String.format("The %s (%x) constant of the " +
+                        "enum %s is supposed to represent a bitmap entry but " +
+                        "has more than one bit set.",
+                        element.toString(), element.getValue(), type.getName());
+                throw new IllegalArgumentException(msg);
+            }
+            allSetBitmap |= element.getValue();
+            if ((bitmap & element.getValue()) != 0) {
+                s.add(element);
+            }
+        }
+        if (((~allSetBitmap) & bitmap) != 0) {
+            // check if only valid flags are set in the given bitmap
+            String msg = String.format("The bitmap %x for enum %s has " +
+                    "bits set that are presented by any enum constant",
+                    bitmap, type.getName());
+            throw new IllegalArgumentException(msg);
+        }
+        return s;
+    }
+
+    /**
+     * Return the bitmap mask with all possible bits set. E.g., If a bitmap
+     * has the individual flags 0x1, 0x2, and 0x8 (note the missing 0x4) then
+     * the mask will be 0xb (1011 binary)
+     *
+     * @param type The Enum class to use. Must implement BitmapableEnum
+     * @throws NullPointerException if type is null
+     * @throws IllegalArgumentException if any enum constant from type has
+     * more than one bit set
+     * @return an integer with all possible bits for the given bitmap enum
+     * type set.
+     */
+    public static <E extends Enum<E> & BitmapableEnum>
+            int getMask(Class<E> type) {
+        if (type == null) {
+            throw new NullPointerException("Given enum type must not be null");
+        }
+        // allSetBitmap will eventually have all valid bits for the given
+        // type set.
+        int allSetBitmap = 0;
+        for (E element: type.getEnumConstants()) {
+            if (Integer.bitCount(element.getValue()) != 1) {
+                String msg = String.format("The %s (%x) constant of the " +
+                        "enum %s is supposed to represent a bitmap entry but " +
+                        "has more than one bit set.",
+                        element.toString(), element.getValue(), type.getName());
+                throw new IllegalArgumentException(msg);
+            }
+            allSetBitmap |= element.getValue();
+        }
+        return allSetBitmap;
+    }
+
+    /**
+     * Convert the given EnumSet to the integer bitmap representation.
+     * @param set The EnumSet to convert. The enum must implement
+     * BitmapableEnum
+     * @return the integer bitmap
+     * @throws IllegalArgumentException if an enum constant from the set (!) has
+     * more than one bit set
+     * @throws NullPointerException if the set is null
+     */
+    public static <E extends Enum<E> & BitmapableEnum>
+            int toBitmap(Set<E> set) {
+        if (set == null) {
+            throw new NullPointerException("Given set must not be null");
+        }
+        int bitmap = 0;
+        for (E element: set) {
+            if (Integer.bitCount(element.getValue()) != 1) {
+                String msg = String.format("The %s (%x) constant in the set " +
+                        "is supposed to represent a bitmap entry but " +
+                        "has more than one bit set.",
+                        element.toString(), element.getValue());
+                throw new IllegalArgumentException(msg);
+            }
+            bitmap |= element.getValue();
+        }
+        return bitmap;
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/FilterIterator.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/FilterIterator.java
new file mode 100644
index 0000000..fdde82c
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/FilterIterator.java
@@ -0,0 +1,85 @@
+/**
+ *    Copyright 2012, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * An iterator that will filter values from an iterator and return only
+ * those values that match the predicate.
+ */
+public abstract class FilterIterator<T> implements Iterator<T> {
+    protected Iterator<T> subIterator;
+    protected T next;
+
+    /**
+     * Construct a filter iterator from the given sub iterator.
+     *
+     * @param subIterator the sub iterator over which we'll filter
+     */
+    public FilterIterator(Iterator<T> subIterator) {
+        super();
+        this.subIterator = subIterator;
+    }
+
+    /**
+     * Check whether the given value should be returned by the
+     * filter.
+     *
+     * @param value the value to check
+     * @return true if the value should be included
+     */
+    protected abstract boolean matches(T value);
+
+    // ***********
+    // Iterator<T>
+    // ***********
+
+    @Override
+    public boolean hasNext() {
+        if (next != null) {
+            return true;
+        }
+
+        while (subIterator.hasNext()) {
+            next = subIterator.next();
+            if (matches(next)) {
+                return true;
+            }
+        }
+        next = null;
+        return false;
+    }
+
+    @Override
+    public T next() {
+        if (hasNext()) {
+            T cur = next;
+            next = null;
+            return cur;
+        }
+        throw new NoSuchElementException();
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/InstanceId.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/InstanceId.java
new file mode 100644
index 0000000..861dec6
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/InstanceId.java
@@ -0,0 +1,47 @@
+package net.onrc.onos.of.ctl.util;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * The class representing an ONOS Instance ID.
+ *
+ * This class is immutable.
+ */
+public final class InstanceId {
+    private final String id;
+
+    /**
+     * Constructor from a string value.
+     *
+     * @param id the value to use.
+     */
+    public InstanceId(String id) {
+        this.id = checkNotNull(id);
+        checkArgument(!id.isEmpty(), "Empty ONOS Instance ID");
+    }
+
+    @Override
+    public int hashCode() {
+        return id.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        }
+
+        if (!(obj instanceof InstanceId)) {
+            return false;
+        }
+
+        InstanceId that = (InstanceId) obj;
+        return this.id.equals(that.id);
+    }
+
+    @Override
+    public String toString() {
+        return id;
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/IterableIterator.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/IterableIterator.java
new file mode 100644
index 0000000..79f3c9d
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/IterableIterator.java
@@ -0,0 +1,68 @@
+/**
+ *    Copyright 2012 Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Iterator over all values in an iterator of iterators.
+ *
+ * @param <T> the type of elements returned by this iterator
+ */
+public class IterableIterator<T> implements Iterator<T> {
+    Iterator<? extends Iterable<T>> subIterator;
+    Iterator<T> current = null;
+
+    public IterableIterator(Iterator<? extends Iterable<T>> subIterator) {
+        super();
+        this.subIterator = subIterator;
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (current == null) {
+            if (subIterator.hasNext()) {
+                current = subIterator.next().iterator();
+            } else {
+                return false;
+            }
+        }
+        while (!current.hasNext() && subIterator.hasNext()) {
+            current = subIterator.next().iterator();
+        }
+
+        return current.hasNext();
+    }
+
+    @Override
+    public T next() {
+        if (hasNext()) {
+            return current.next();
+        }
+        throw new NoSuchElementException();
+    }
+
+    @Override
+    public void remove() {
+        if (hasNext()) {
+            current.remove();
+        }
+        throw new NoSuchElementException();
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/LRUHashMap.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/LRUHashMap.java
new file mode 100644
index 0000000..17f9354
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/LRUHashMap.java
@@ -0,0 +1,38 @@
+/**
+ *    Copyright 2011, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.util;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class LRUHashMap<K, V> extends LinkedHashMap<K, V> {
+
+    private static final long serialVersionUID = 1L;
+
+    private final int capacity;
+
+    public LRUHashMap(int capacity) {
+        super(capacity + 1, 0.75f, true);
+        this.capacity = capacity;
+    }
+
+    protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+        return size() > capacity;
+    }
+
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/LinkedHashSetWrapper.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/LinkedHashSetWrapper.java
new file mode 100644
index 0000000..629e536
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/LinkedHashSetWrapper.java
@@ -0,0 +1,32 @@
+package net.onrc.onos.of.ctl.util;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+
+import com.google.common.collect.ForwardingCollection;
+
+/**
+ * A simple wrapper / forwarder that forwards all calls to a LinkedHashSet.
+ * This wrappers sole reason for existence is to implement the
+ * OrderedCollection marker interface.
+ *
+ */
+public class LinkedHashSetWrapper<E>
+        extends ForwardingCollection<E> implements OrderedCollection<E> {
+    private final Collection<E> delegate;
+
+    public LinkedHashSetWrapper() {
+        super();
+        this.delegate = new LinkedHashSet<E>();
+    }
+
+    public LinkedHashSetWrapper(Collection<? extends E> c) {
+        super();
+        this.delegate = new LinkedHashSet<E>(c);
+    }
+
+    @Override
+    protected Collection<E> delegate() {
+        return this.delegate;
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/MultiIterator.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/MultiIterator.java
new file mode 100644
index 0000000..693a8bf
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/MultiIterator.java
@@ -0,0 +1,68 @@
+/**
+ *    Copyright 2012 Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.onrc.onos.of.ctl.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Iterator over all values in an iterator of iterators.
+ *
+ * @param <T> the type of elements returned by this iterator
+ */
+public class MultiIterator<T> implements Iterator<T> {
+    Iterator<Iterator<T>> subIterator;
+    Iterator<T> current = null;
+
+    public MultiIterator(Iterator<Iterator<T>> subIterator) {
+        super();
+        this.subIterator = subIterator;
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (current == null) {
+            if (subIterator.hasNext()) {
+                current = subIterator.next();
+            } else {
+                return false;
+            }
+        }
+        while (!current.hasNext() && subIterator.hasNext()) {
+            current = subIterator.next();
+        }
+
+        return current.hasNext();
+    }
+
+    @Override
+    public T next() {
+        if (hasNext()) {
+            return current.next();
+        }
+        throw new NoSuchElementException();
+    }
+
+    @Override
+    public void remove() {
+        if (hasNext()) {
+            current.remove();
+        }
+        throw new NoSuchElementException();
+    }
+}
diff --git a/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/OrderedCollection.java b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/OrderedCollection.java
new file mode 100644
index 0000000..f032212
--- /dev/null
+++ b/of/ctl/src/main/java/net/onrc/onos/of/ctl/util/OrderedCollection.java
@@ -0,0 +1,14 @@
+package net.onrc.onos.of.ctl.util;
+
+import java.util.Collection;
+
+/**
+ * A marker interface indicating that this Collection defines a particular
+ * iteration order. The details about the iteration order are specified by
+ * the concrete implementation.
+ *
+ * @param <E>
+ */
+public interface OrderedCollection<E> extends Collection<E> {
+
+}