diff --git a/src/main/java/net/floodlightcontroller/core/FloodlightProvider.java b/src/main/java/net/floodlightcontroller/core/FloodlightProvider.java
index 4183877..0ee2904 100644
--- a/src/main/java/net/floodlightcontroller/core/FloodlightProvider.java
+++ b/src/main/java/net/floodlightcontroller/core/FloodlightProvider.java
@@ -1,3 +1,19 @@
+/**
+ *    Copyright 2013, Big Switch Networks, Inc.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
 package net.floodlightcontroller.core;
 
 import java.util.ArrayList;
@@ -10,6 +26,8 @@
 import net.floodlightcontroller.core.module.FloodlightModuleException;
 import net.floodlightcontroller.core.module.IFloodlightModule;
 import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.debugcounter.IDebugCounterService;
+import net.floodlightcontroller.debugevent.IDebugEventService;
 import net.floodlightcontroller.restserver.IRestApiService;
 import net.floodlightcontroller.threadpool.IThreadPoolService;
 import net.onrc.onos.core.linkdiscovery.ILinkDiscoveryService;
@@ -28,13 +46,13 @@
 
     @Override
     public Map<Class<? extends IFloodlightService>,
-            IFloodlightService> getServiceImpls() {
+               IFloodlightService> getServiceImpls() {
         controller = new Controller();
 
         Map<Class<? extends IFloodlightService>,
-                IFloodlightService> m =
+            IFloodlightService> m =
                 new HashMap<Class<? extends IFloodlightService>,
-                        IFloodlightService>();
+                            IFloodlightService>();
         m.put(IFloodlightProviderService.class, controller);
         return m;
     }
@@ -42,33 +60,36 @@
     @Override
     public Collection<Class<? extends IFloodlightService>> getModuleDependencies() {
         Collection<Class<? extends IFloodlightService>> dependencies =
-                new ArrayList<Class<? extends IFloodlightService>>(4);
+            new ArrayList<Class<? extends IFloodlightService>>(4);
         dependencies.add(IRestApiService.class);
+        dependencies.add(IDebugCounterService.class);
+        dependencies.add(IDebugEventService.class);
         dependencies.add(IThreadPoolService.class);
-        // Following added by ONOS
-        // dependencies.add(IControllerRegistryService.class);
-        // dependencies.add(ILinkDiscoveryService.class);
-
         return dependencies;
     }
 
     @Override
     public void init(FloodlightModuleContext context) throws FloodlightModuleException {
-        controller.setRestApiService(
-                context.getServiceImpl(IRestApiService.class));
-        controller.setThreadPoolService(
-                context.getServiceImpl(IThreadPoolService.class));
-        // Following added by ONOS
-        controller.setMastershipService(
-                context.getServiceImpl(IControllerRegistryService.class));
-        controller.setLinkDiscoveryService(
-                context.getServiceImpl(ILinkDiscoveryService.class));
+       controller.setDebugCounter(
+           context.getServiceImpl(IDebugCounterService.class));
+       controller.setDebugEvent(
+           context.getServiceImpl(IDebugEventService.class));
+       controller.setRestApiService(
+           context.getServiceImpl(IRestApiService.class));
+       controller.setThreadPoolService(
+           context.getServiceImpl(IThreadPoolService.class));
+       // Following added by ONOS
+       controller.setMastershipService(
+               context.getServiceImpl(IControllerRegistryService.class));
+       controller.setLinkDiscoveryService(
+               context.getServiceImpl(ILinkDiscoveryService.class));
 
-        controller.init(context.getConfigParams(this));
+       controller.init(context.getConfigParams(this));
     }
 
     @Override
-    public void startUp(FloodlightModuleContext context) {
+    public void startUp(FloodlightModuleContext context)
+            throws FloodlightModuleException {
         controller.startupComponents();
     }
 }
diff --git a/src/main/java/net/floodlightcontroller/core/IFloodlightProviderService.java b/src/main/java/net/floodlightcontroller/core/IFloodlightProviderService.java
index 1194e18..7f4f77d 100644
--- a/src/main/java/net/floodlightcontroller/core/IFloodlightProviderService.java
+++ b/src/main/java/net/floodlightcontroller/core/IFloodlightProviderService.java
@@ -1,5 +1,5 @@
 /**
- *    Copyright 2011, Big Switch Networks, Inc. 
+ *    Copyright 2011, Big Switch Networks, Inc.
  *    Originally created by David Erickson, Stanford University
  *
  *    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -19,15 +19,16 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
+import net.floodlightcontroller.core.internal.Controller.Counters;
 import net.floodlightcontroller.core.module.IFloodlightService;
-import net.onrc.onos.api.registry.ILocalSwitchMastershipListener;
 import net.onrc.onos.core.packet.Ethernet;
 import net.onrc.onos.core.util.OnosInstanceId;
 
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFType;
-import org.openflow.protocol.factory.BasicFactory;
+import org.projectfloodlight.openflow.protocol.OFControllerRole;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFType;
 
 /**
  * The interface exposed by the core bundle that allows you to interact
@@ -43,13 +44,41 @@
      */
     public static final String CONTEXT_PI_PAYLOAD =
             "net.floodlightcontroller.core.IFloodlightProvider.piPayload";
-
+    public static final String CONTEXT_PI_INPORT =
+            "net.floodlightcontroller.core.IFloodlightProvider.piInPort";;
     /**
-     * The role of the controller as used by the OF 1.2 and OVS failover and
-     * load-balancing mechanism.
+     * The role of the controller as it pertains to a particular switch.
+     * Note that this definition of the role enum is different from the
+     * OF1.3 definition. It is maintained here to be backward compatible to
+     * earlier versions of the controller code. This enum is translated
+     * to the OF1.3 enum, before role messages are sent to the switch.
+     * See sendRoleRequestMessage method in OFSwitchImpl
      */
     public static enum Role {
-        EQUAL, MASTER, SLAVE
+        EQUAL(OFControllerRole.ROLE_EQUAL),
+        MASTER(OFControllerRole.ROLE_MASTER),
+        SLAVE(OFControllerRole.ROLE_SLAVE);
+
+        private final int nxRole;
+
+        private Role(OFControllerRole nxRole) {
+            this.nxRole = nxRole.ordinal();
+        }
+        /*
+        private static Map<Integer,Role> nxRoleToEnum
+                = new HashMap<Integer,Role>();
+        static {
+            for(Role r: Role.values())
+                nxRoleToEnum.put(r.toNxRole(), r);
+        }
+        public int toNxRole() {
+            return nxRole;
+        }
+        // Return the enum representing the given nxRole or null if no
+        // such role exists
+        public static Role fromNxRole(int nxRole) {
+            return nxRoleToEnum.get(nxRole);
+        }*/
     }
 
     /**
@@ -59,6 +88,40 @@
     public static final FloodlightContextStore<Ethernet> bcStore =
             new FloodlightContextStore<Ethernet>();
 
+
+    //************************
+    //  Controller related
+    //************************
+
+    /**
+     * Get the current mapping of controller IDs to their IP addresses
+     * Returns a copy of the current mapping.
+     *
+     * @see IHAListener
+     */
+    public Map<String, String> getControllerNodeIPs();
+
+    /**
+     * Return the controller start time in  milliseconds
+     *
+     * @return
+     */
+    public long getSystemStartTime();
+
+    /**
+     * Run the main I/O loop of the Controller.
+     */
+    public void run();
+
+//    /**
+//     * Terminate the process
+//     */
+//    public void terminate();
+
+    //************************
+    //  OF Message Listener related
+    //************************
+
     /**
      * Adds an OpenFlow message listener
      *
@@ -82,23 +145,9 @@
      */
     public Map<OFType, List<IOFMessageListener>> getListeners();
 
-    /**
-     * Adds a switch mastership listener for controller role changes for
-     * local switches.
-     *
-     * @param listener the listener to add.
-     */
-    public void addLocalSwitchMastershipListener(
-                ILocalSwitchMastershipListener listener);
-
-    /**
-     * Removes a switch mastership listener for controller role changes for
-     * local switches.
-     *
-     * @param listener the listener to remove.
-     */
-    public void removeLocalSwitchMastershipListener(
-                ILocalSwitchMastershipListener listener);
+    //************************
+    //  Switch & SwitchListener related
+    //************************
 
     /**
      * Returns an unmodifiable map of all actively connected OpenFlow switches. This doesn't
@@ -109,12 +158,12 @@
     public Map<Long, IOFSwitch> getSwitches();
 
     /**
-     * Get the current mapping of controller IDs to their IP addresses
-     * Returns a copy of the current mapping.
-     *
-     * @see IHAListener
+     * Configure controller to always clear the flow table on the switch,
+     * when it connects to controller. This will be true for first time switch
+     * reconnect, as well as a switch re-attaching to Controller after HA
+     * switch over to ACTIVE role
      */
-    public Map<String, String> getControllerNodeIPs();
+    public void setAlwaysClearFlowsOnSwAdd(boolean value);
 
     /**
      * Gets the unique ID used to identify this ONOS instance in the cluster.
@@ -137,67 +186,19 @@
      */
     public void removeOFSwitchListener(IOFSwitchListener listener);
 
-    /**
-     * Terminate the process
-     */
-    public void terminate();
+
+
+    //************************
+    //  Utility methods
+    //************************
 
     /**
-     * Re-injects an OFMessage back into the packet processing chain
-     *
-     * @param sw  The switch to use for the message
-     * @param msg the message to inject
-     * @return True if successfully re-injected, false otherwise
-     */
-    public boolean injectOfMessage(IOFSwitch sw, OFMessage msg);
-
-    /**
-     * Re-injects an OFMessage back into the packet processing chain
-     *
-     * @param sw       The switch to use for the message
-     * @param msg      the message to inject
-     * @param bContext a floodlight context to use if required
-     * @return True if successfully re-injected, false otherwise
-     */
-    public boolean injectOfMessage(IOFSwitch sw, OFMessage msg,
-                                   FloodlightContext bContext);
-
-    /**
-     * Process written messages through the message listeners for the controller
-     *
-     * @param sw The switch being written to
-     * @param m  the message
-     * @param bc any accompanying context object
-     */
-    public void handleOutgoingMessage(IOFSwitch sw, OFMessage m,
-                                      FloodlightContext bc);
-
-    /**
-     * Gets the BasicFactory
+     * Gets the Factory
      *
      * @return an OpenFlow message factory
      */
-    public BasicFactory getOFMessageFactory();
-
-    /**
-     * Run the main I/O loop of the Controller.
-     */
-    public void run();
-
-    /**
-     * Return the controller start time in  milliseconds
-     *
-     * @return
-     */
-    public long getSystemStartTime();
-
-    /**
-     * Configure controller to always clear the flow table on the switch,
-     * when it connects to controller. This will be true for first time switch
-     * reconnect, as well as a switch re-attaching to Controller after HA
-     * switch over to ACTIVE role
-     */
-    public void setAlwaysClearFlowsOnSwAdd(boolean value);
+    public OFFactory getOFMessageFactory_13();
+    public OFFactory getOFMessageFactory_10();
 
     /**
      * Publish updates to Controller updates queue
@@ -206,4 +207,66 @@
      */
     public void publishUpdate(IUpdate update);
 
+//    /**
+//     * Re-injects an OFMessage back into the packet processing chain
+//     *
+//     * @param sw  The switch to use for the message
+//     * @param msg the message to inject
+//     * @return True if successfully re-injected, false otherwise
+//     */
+//    public boolean injectOfMessage(IOFSwitch sw, OFMessage msg);
+//
+//    /**
+//     * Re-injects an OFMessage back into the packet processing chain
+//     *
+//     * @param sw       The switch to use for the message
+//     * @param msg      the message to inject
+//     * @param bContext a floodlight context to use if required
+//     * @return True if successfully re-injected, false otherwise
+//     */
+//    public boolean injectOfMessage(IOFSwitch sw, OFMessage msg,
+//                                   FloodlightContext bContext);
+//
+//    /**
+//     * Process written messages through the message listeners for the controller
+//     *
+//     * @param sw The switch being written to
+//     * @param m  the message
+//     * @param bc any accompanying context object
+//     */
+//    public void handleOutgoingMessage(IOFSwitch sw, OFMessage m,
+//                                      FloodlightContext bc);
+
+
+    /**
+     * Return the default set of counters
+     * @return
+     */
+    public Counters getCounters();
+
+    void setAlwaysClearFlowsOnSwActivate(boolean value);
+
+    Map<String, Long> getMemory();
+
+    Long getUptime();
+
+    Set<Long> getAllSwitchDpids();
+
+    IOFSwitch getSwitch(long dpid);
+
+    /**
+     * Record a switch event in in-memory debug-event
+     * @param switchDPID
+     * @param reason Reason for this event
+     * @param flushNow see debug-event flushing in IDebugEventService
+     */
+    public void addSwitchEvent(long switchDPID, String reason, boolean flushNow);
+
+    Set<Long> getAllMasterSwitchDpids();
+
+    Set<Long> getAllEqualSwitchDpids();
+
+    IOFSwitch getMasterSwitch(long dpid);
+
+    IOFSwitch getEqualSwitch(long dpid);
 }
diff --git a/src/main/java/net/floodlightcontroller/core/IOFMessageListener.java b/src/main/java/net/floodlightcontroller/core/IOFMessageListener.java
index b14af57..3bf7988 100644
--- a/src/main/java/net/floodlightcontroller/core/IOFMessageListener.java
+++ b/src/main/java/net/floodlightcontroller/core/IOFMessageListener.java
@@ -17,8 +17,8 @@
 
 package net.floodlightcontroller.core;
 
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFType;
 
 /**
  * @author David Erickson (daviderickson@cs.stanford.edu)
diff --git a/src/main/java/net/floodlightcontroller/core/IOFSwitch.java b/src/main/java/net/floodlightcontroller/core/IOFSwitch.java
index e28d7be..54e4408 100644
--- a/src/main/java/net/floodlightcontroller/core/IOFSwitch.java
+++ b/src/main/java/net/floodlightcontroller/core/IOFSwitch.java
@@ -22,32 +22,52 @@
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.Future;
 
 import net.floodlightcontroller.core.IFloodlightProviderService.Role;
+import net.floodlightcontroller.debugcounter.IDebugCounterService;
+import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterException;
+import net.floodlightcontroller.threadpool.IThreadPoolService;
+import net.floodlightcontroller.util.OrderedCollection;
 
 import org.jboss.netty.channel.Channel;
-import org.openflow.protocol.OFFeaturesReply;
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFPhysicalPort;
-import org.openflow.protocol.OFStatisticsRequest;
-import org.openflow.protocol.statistics.OFDescriptionStatistics;
-import org.openflow.protocol.statistics.OFStatistics;
+import org.projectfloodlight.openflow.protocol.OFActionType;
+import org.projectfloodlight.openflow.protocol.OFCapabilities;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFFeaturesReply;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFPortStatus;
+import org.projectfloodlight.openflow.protocol.OFStatsReply;
+import org.projectfloodlight.openflow.protocol.OFStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.types.U64;
 
-/**
- * @author David Erickson (daviderickson@cs.stanford.edu)
- */
+
 public interface IOFSwitch {
-    // Attribute keys
-    public static final String SWITCH_DESCRIPTION_FUTURE = "DescriptionFuture";
-    public static final String SWITCH_DESCRIPTION_DATA = "DescriptionData";
+
+	/**
+	 * OF1.3 switches should support role-request messages as in the 1.3 spec.
+     * OF1.0 switches may or may not support the Nicira role request extensions.
+     * To indicate the support, this property should be set by the associated
+	 * OF1.0 switch driver in the net.onrc.onos.core.drivermanager package.
+	 * The property will be ignored for OF1.3 switches.
+	 */
     public static final String SWITCH_SUPPORTS_NX_ROLE = "supportsNxRole";
-    public static final String SWITCH_IS_CORE_SWITCH = "isCoreSwitch";
-    public static final String PROP_FASTWILDCARDS = "FastWildcards";
-    public static final String PROP_REQUIRES_L3_MATCH = "requiresL3Match";
-    public static final String PROP_SUPPORTS_OFPP_TABLE = "supportsOfppTable";
-    public static final String PROP_SUPPORTS_OFPP_FLOOD = "supportsOfppFlood";
-    public static final String PROP_SUPPORTS_NETMASK_TBL = "supportsNetmaskTbl";
+
+
+    //************************
+    // Channel related
+    //************************
+
+    /**
+     * Disconnects the switch by closing the TCP connection. Results in a call
+     * to the channel handler's channelDisconnected method for cleanup
+     * @throws IOException
+     */
+    public void disconnectSwitch();
 
     /**
      * Writes to the OFMessage to the output stream.
@@ -61,7 +81,7 @@
     public void write(OFMessage m, FloodlightContext bc) throws IOException;
 
     /**
-     * Writes the list of messages to the output stream
+     * Writes the list of messages to the output stream.
      * The message will be handed to the floodlightProvider for possible filtering
      * and processing by message listeners.
      *
@@ -72,45 +92,210 @@
     public void write(List<OFMessage> msglist, FloodlightContext bc) throws IOException;
 
     /**
-     * @throws IOException
-     */
-    public void disconnectOutputStream();
-
-    /**
-     * FIXME: remove getChannel(). All access to the channel should be through
-     * wrapper functions in IOFSwitch
+     * Gets the date the switch connected to this controller.
      *
-     * @return the channel to the switch
+     * @return the date
      */
-    public Channel getChannel();
+    public Date getConnectedSince();
 
     /**
-     * Returns switch features from features Reply
+     * Gets the next available transaction id.
+     *
+     * @return the next transaction ID
+     */
+    public int getNextTransactionId();
+
+    /**
+     * Checks if the switch is still connected.
+     * Only call while holding processMessageLock
+     *
+     * @return whether the switch is still disconnected
+     */
+    public boolean isConnected();
+
+    /**
+     * Sets whether the switch is connected.
+     * Only call while holding modifySwitchLock
+     *
+     * @param connected whether the switch is connected
+     */
+    public void setConnected(boolean connected);
+
+    /**
+     * Flushes all flows queued for this switch in the current thread.
+     * NOTE: The contract is limited to the current thread
+     */
+    public void flush();
+
+    /**
+     * Sets the Netty Channel this switch instance is associated with.
+     * <p>
+     * Called immediately after instantiation
+     *
+     * @param channel
+     */
+    public void setChannel(Channel channel);
+
+    //************************
+    // Switch features related
+    //************************
+
+    /**
+     * Gets the datapathId of the switch.
      *
      * @return the switch buffers
      */
-    public int getBuffers();
-
-    public int getActions();
-
-    public int getCapabilities();
-
-    public byte getTables();
+    public long getId();
 
     /**
-     * Set the OFFeaturesReply message returned by the switch during initial
-     * handshake.
+     * Gets a string version of the ID for this switch.
      *
-     * @param featuresReply
+     * @return string version of the ID
      */
-    public void setFeaturesReply(OFFeaturesReply featuresReply);
+    public String getStringId();
 
     /**
-     * Set the SwitchProperties based on it's description
+     * Gets the number of buffers.
      *
-     * @param description
+     * @return the number of buffers
      */
-    public void setSwitchProperties(OFDescriptionStatistics description);
+    public int getNumBuffers();
+
+    public Set<OFCapabilities> getCapabilities();
+
+    public byte getNumTables();
+
+    /**
+     * Returns an OFDescStatsReply message object. Use the methods contained
+     * to retrieve switch descriptions for Manufacturer, Hw/Sw version etc.
+     */
+    public OFDescStatsReply getSwitchDescription();
+
+//    /**
+//     * Returns a Future object that can be used to retrieve the asynchronous
+//     * OFStatisticsReply when it is available.
+//     *
+//     * @param request statistics request
+//     * @return Future object wrapping OFStatisticsReply
+//     * @throws IOException
+//     */
+//    public Future<OFFeaturesReply> getFeaturesReplyFromSwitch()
+//            throws IOException;
+//
+//    /**
+//     * Deliver the featuresReply future reply
+//     *
+//     * @param reply the reply to deliver
+//     */
+//    void deliverOFFeaturesReply(OFMessage reply);
+
+    /**
+     * Cancel features reply with a specific transaction ID
+     * @param transactionId the transaction ID
+     */
+    public void cancelFeaturesReply(int transactionId);
+
+    /**
+     * Gets the OFActionType set.
+     * <p>
+     * getActions has relevance only for an OpenFlow 1.0 switch.
+     * For OF1.3, each table can support different actions
+     *
+     * @return the action set
+     */
+    public Set<OFActionType> getActions();
+
+    public void setOFVersion(OFVersion ofv);
+
+    public OFVersion getOFVersion();
+
+
+    //************************
+    //  Switch port related
+    //************************
+
+    /**
+     * the type of change that happened to an open flow port
+     */
+    public enum PortChangeType {
+        /** Either a new port has been added by the switch, or we are
+         * adding a port we just deleted (via a prior notification) due to
+         * a change in the portNumber-portName mapping.
+         */
+        ADD,
+	    /** some other feature of the port has changed (eg. speed)*/
+        OTHER_UPDATE,
+        /** Either a port has been deleted by the switch, or we are deleting
+         * a port whose portNumber-portName mapping has changed. Note that in
+         * the latter case, a subsequent notification will be sent out to add a
+         * port with the new portNumber-portName mapping.
+         */
+        DELETE,
+        /** Port is up (i.e. enabled). Presumably an earlier notification had
+         * indicated that it was down. To be UP implies that the port is
+         * administratively considered UP (see ofp_port_config) AND the port
+         * link is up AND the port is no longer blocked (see ofp_port_state).
+         */
+        UP,
+        /** Port is down (i.e. disabled). Presumably an earlier notification had
+         * indicated that it was up, or the port was always up.
+         * To be DOWN implies that the port has been either
+         * administratively brought down (see ofp_port_config) OR the port
+         * link is down OR the port is blocked (see ofp_port_state).
+         */
+        DOWN,
+    }
+
+    /**
+     * Describes a change of an open flow port.
+     */
+    public static class PortChangeEvent {
+        public final OFPortDesc port;
+        public final PortChangeType type;
+        /**
+         * @param port
+         * @param type
+         */
+        public PortChangeEvent(OFPortDesc port,
+                               PortChangeType type) {
+            this.port = port;
+            this.type = type;
+        }
+        /* (non-Javadoc)
+         * @see java.lang.Object#hashCode()
+         */
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + ((port == null) ? 0 : port.hashCode());
+            result = prime * result + ((type == null) ? 0 : type.hashCode());
+            return result;
+        }
+        /* (non-Javadoc)
+         * @see java.lang.Object#equals(java.lang.Object)
+         */
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) return true;
+            if (obj == null) return false;
+            if (getClass() != obj.getClass()) return false;
+            PortChangeEvent other = (PortChangeEvent) obj;
+            if (port == null) {
+                if (other.port != null) return false;
+            } else if (!port.equals(other.port)) return false;
+            if (type != other.type) return false;
+            return true;
+        }
+        /* (non-Javadoc)
+         * @see java.lang.Object#toString()
+         */
+        @Override
+        public String toString() {
+            return "[" + type + " " + port.toString() + "]";
+        }
+    }
+
 
     /**
      * Get list of all enabled ports. This will typically be different from
@@ -121,7 +306,7 @@
      *
      * @return Unmodifiable list of ports not backed by the underlying collection
      */
-    public Collection<OFPhysicalPort> getEnabledPorts();
+    public Collection<OFPortDesc> getEnabledPorts();
 
     /**
      * Get list of the port numbers of all enabled ports. This will typically
@@ -132,7 +317,7 @@
      *
      * @return Unmodifiable list of ports not backed by the underlying collection
      */
-    public Collection<Short> getEnabledPortNumbers();
+    public Collection<Integer> getEnabledPortNumbers();
 
     /**
      * Retrieve the port object by the port number. The port object
@@ -142,7 +327,7 @@
      * @param portNumber
      * @return port object
      */
-    public OFPhysicalPort getPort(short portNumber);
+    public OFPortDesc getPort(int portNumber);
 
     /**
      * Retrieve the port object by the port name. The port object
@@ -152,34 +337,28 @@
      * @param portName
      * @return port object
      */
-    public OFPhysicalPort getPort(String portName);
+    public OFPortDesc getPort(String portName);
 
     /**
      * Add or modify a switch port. This is called by the core controller
      * code in response to a OFPortStatus message. It should not typically be
      * called by other floodlight applications.
      *
-     * @param port
-     */
-    public void setPort(OFPhysicalPort port);
-
-    /**
-     * Delete a port for the switch. This is called by the core controller
-     * code in response to a OFPortStatus message. It should not typically be
-     * called by other floodlight applications.
+     * OFPPR_MODIFY and OFPPR_ADD will be treated as equivalent. The OpenFlow
+     * spec is not clear on whether portNames are portNumbers are considered
+     * authoritative identifiers. We treat portNames <-> portNumber mappings
+     * as fixed. If they change, we delete all previous conflicting ports and
+     * add all new ports.
      *
-     * @param portNumber
+     * @param ps the port status message
+     * @return the ordered Collection of changes "applied" to the old ports
+     * of the switch according to the PortStatus message. A single PortStatus
+     * message can result in multiple changes.
+     * If portName <-> portNumber mappings have
+     * changed, the iteration order ensures that delete events for old
+     * conflicting appear before before events adding new ports
      */
-    public void deletePort(short portNumber);
-
-    /**
-     * Delete a port for the switch. This is called by the core controller
-     * code in response to a OFPortStatus message. It should not typically be
-     * called by other floodlight applications.
-     *
-     * @param portName
-     */
-    public void deletePort(String portName);
+    public OrderedCollection<PortChangeEvent> processOFPortStatus(OFPortStatus ps);
 
     /**
      * Get list of all ports. This will typically be different from
@@ -190,14 +369,14 @@
      *
      * @return Unmodifiable list of ports
      */
-    public Collection<OFPhysicalPort> getPorts();
+    public Collection<OFPortDesc> getPorts();
 
     /**
      * @param portName
      * @return Whether a port is enabled per latest port status message
      * (not configured down nor link down nor in spanning tree blocking state)
      */
-    public boolean portEnabled(short portName);
+    public boolean portEnabled(int portName);
 
     /**
      * @param portNumber
@@ -207,150 +386,82 @@
     public boolean portEnabled(String portName);
 
     /**
-     * @param port
-     * @return Whether a port is enabled per latest port status message
-     * (not configured down nor link down nor in spanning tree blocking state)
+     * Compute the changes that would be required to replace the old ports
+     * of this switch with the new ports
+     * @param ports new ports to set
+     * @return the ordered collection of changes "applied" to the old ports
+     * of the switch in order to set them to the new set.
+     * If portName <-> portNumber mappings have
+     * changed, the iteration order ensures that delete events for old
+     * conflicting appear before before events adding new ports
      */
-    public boolean portEnabled(OFPhysicalPort port);
+    public OrderedCollection<PortChangeEvent>
+            comparePorts(Collection<OFPortDesc> ports);
 
     /**
-     * Get the datapathId of the switch
-     *
-     * @return switch dpid as a long
+     * Replace the ports of this switch with the given ports.
+     * @param ports new ports to set
+     * @return the ordered collection of changes "applied" to the old ports
+     * of the switch in order to set them to the new set.
+     * If portName <-> portNumber mappings have
+     * changed, the iteration order ensures that delete events for old
+     * conflicting appear before before events adding new ports
      */
-    public long getId();
+    public OrderedCollection<PortChangeEvent>
+            setPorts(Collection<OFPortDesc> ports);
+
+//  XXX S The odd use of providing an API call to 'set ports' (above) would
+//  logically suggest that there should be a way to delete or unset the ports.
+//  Right now we forbid this. We should probably not use setPorts too.
+//
+//  /**
+//   * Delete a port for the switch. This is called by the core controller
+//   * code in response to a OFPortStatus message. It should not typically be
+//   * called by other floodlight applications.
+//   *
+//   * @param portNumber
+//   */
+//  public void deletePort(short portNumber);
+//
+//  /**
+//   * Delete a port for the switch. This is called by the core controller
+//   * code in response to a OFPortStatus message. It should not typically be
+//   * called by other floodlight applications.
+//   *
+//   * @param portName
+//   */
+//  public void deletePort(String portName);
+
+
+    //*******************************************
+    //  IOFSwitch object attributes
+    //************************
 
     /**
-     * Get a string version of the ID for this switch
-     *
-     * @return switch dpid as a String
-     */
-    public String getStringId();
-
-    /**
-     * Retrieves attributes of this switch
+     * Gets attributes of this switch.
      *
      * @return attributes of the switch
      */
     public Map<Object, Object> getAttributes();
 
-    /**
-     * Retrieves the date the switch connected to this controller
-     *
-     * @return the date
-     */
-    public Date getConnectedSince();
-
-    /**
-     * Returns the next available transaction id
-     *
-     * @return next available transaction id
-     */
-    public int getNextTransactionId();
-
-    /**
-     * Returns a Future object that can be used to retrieve the asynchronous
-     * OFStatisticsReply when it is available.
-     *
-     * @param request statistics request
-     * @return Future object wrapping OFStatisticsReply
-     * @throws IOException
-     */
-    public Future<List<OFStatistics>> getStatistics(OFStatisticsRequest request)
-            throws IOException;
-
-    /**
-     * Returns a Future object that can be used to retrieve the asynchronous
-     * OFStatisticsReply when it is available.
-     *
-     * @param request statistics request
-     * @return Future object wrapping OFStatisticsReply
-     * @throws IOException
-     */
-    public Future<OFFeaturesReply> getFeaturesReplyFromSwitch()
-            throws IOException;
-
-    /**
-     * Deliver the featuresReply future reply
-     *
-     * @param reply the reply to deliver
-     */
-    void deliverOFFeaturesReply(OFMessage reply);
-
-    /*
-     * Cancel features reply with a specific transction ID
-     * @param transactionId the transaction ID
-     */
-    public void cancelFeaturesReply(int transactionId);
-
-    /**
-     * Check if the switch is still connected;
-     * Only call while holding processMessageLock
-     *
-     * @return whether the switch is still disconnected
-     */
-    public boolean isConnected();
-
-    /**
-     * Set whether the switch is connected
-     * Only call while holding modifySwitchLock
-     *
-     * @param connected whether the switch is connected
-     */
-    public void setConnected(boolean connected);
-
-    /**
-     * Get the current role of the controller for the switch
-     *
-     * @return the role of the controller
-     */
-    public Role getRole();
-
-    /**
-     * Check if the controller is an active controller for the switch.
-     * The controller is active if its role is MASTER or EQUAL.
-     *
-     * @return whether the controller is active
-     */
-    public boolean isActive();
-
-    /**
-     * Deliver the statistics future reply
-     *
-     * @param reply the reply to deliver
-     */
-    public void deliverStatisticsReply(OFMessage reply);
-
-    /**
-     * Cancel the statistics reply with the given transaction ID
-     *
-     * @param transactionId the transaction ID
-     */
-    public void cancelStatisticsReply(int transactionId);
-
-    /**
-     * Cancel all statistics replies
-     */
-    public void cancelAllStatisticsReplies();
-
-    /**
-     * Checks if a specific switch property exists for this switch
-     *
+	/**
+	 * Checks if a specific switch property exists for this switch
+	 *
      * @param name name of property
      * @return value for name
      */
     boolean hasAttribute(String name);
 
     /**
-     * Set properties for switch specific behavior
+     * Gets properties for switch specific behavior.
      *
      * @param name name of property
-     * @return value for name
+     * @return 'value' for 'name', or null if no entry for 'name' exists
      */
     Object getAttribute(String name);
 
     /**
-     * Set properties for switch specific behavior
+     * Sets properties for switch specific behavior.
      *
      * @param name  name of property
      * @param value value for name
@@ -358,53 +469,142 @@
     void setAttribute(String name, Object value);
 
     /**
-     * Set properties for switch specific behavior
+     * Removes properties for switch specific behavior.
      *
      * @param name name of property
      * @return current value for name or null (if not present)
      */
     Object removeAttribute(String name);
 
+    //************************
+    //  Switch statistics
+    //************************
+
     /**
-     * Clear all flowmods on this switch
+     * Delivers the statistics future reply.
+     *
+     * @param reply the reply to deliver
+     */
+    public void deliverStatisticsReply(OFMessage reply);
+
+    /**
+     * Cancels the statistics reply with the given transaction ID.
+     *
+     * @param transactionId the transaction ID
+     */
+    public void cancelStatisticsReply(int transactionId);
+
+    /**
+     * Cancels all statistics replies.
+     */
+    public void cancelAllStatisticsReplies();
+
+    /**
+     * Gets a Future object that can be used to retrieve the asynchronous.
+     * OFStatisticsReply when it is available.
+     *
+     * @param request statistics request
+     * @return Future object wrapping OFStatisticsReply
+     * @throws IOException
+     */
+    public Future<List<OFStatsReply>> getStatistics(OFStatsRequest<?> request)
+            throws IOException;
+
+    //************************
+    //  Switch other utilities
+    //************************
+
+    /**
+     * Clears all flowmods on this switch.
      */
     public void clearAllFlowMods();
 
     /**
-     * Update broadcast cache
-     *
-     * @param data
-     * @return true if there is a cache hit
-     * false if there is no cache hit.
+     * Gets the current role of this controller for this IOFSwitch.
      */
-    public boolean updateBroadcastCache(Long entry, Short port);
+    public Role getRole();
 
     /**
-     * Get the portBroadcastCacheHits
+     * Sets this controller's Role for this IOFSwitch to role.
      *
-     * @return port broadcast cache hits
+     * @param role
      */
-    public Map<Short, Long> getPortBroadcastHits();
+    public void setRole(Role role);
 
     /**
-     * Send a flow statistics request to the switch. This call returns after
-     * sending the stats. request to the switch.
+     * Gets the next generation ID.
+     * <p>
+     * Note: relevant for role request messages in OF1.3
      *
-     * @param request flow statistics request message
-     * @param xid     transaction id, must be obtained by using the getXid() API.
-     * @param caller  the caller of the API. receive() callback of this
-     *                caller would be called when the reply from the switch is received.
-     * @return the transaction id for the message sent to the switch. The
-     * transaction id can be used to match the response with the request. Note
-     * that the transaction id is unique only within the scope of this switch.
-     * @throws IOException
+     * @return next generation ID
      */
-    public void sendStatsQuery(OFStatisticsRequest request, int xid,
-                               IOFMessageListener caller) throws IOException;
+    public U64 getNextGenerationId();
 
     /**
-     * Flush all flows queued for this switch in the current thread.
-     * NOTE: The contract is limited to the current thread
+     * Sets IFloodlightProviderService for this switch instance.
+     * <p>
+     * Called immediately after instantiation
+     *
+     * @param controller
      */
-    public void flush();
+    public void setFloodlightProvider(IFloodlightProviderService controller);
+
+    /**
+     * Sets IThreadPoolService for this switch instance.
+     * <p>
+     * Called immediately after instantiation
+     *
+     * @param threadPool
+     */
+    public void setThreadPoolService(IThreadPoolService threadPool);
+
+    /**
+     * Set debug counter service for per-switch counters
+     * Called immediately after instantiation
+     * @param debugCounters
+     * @throws CounterException
+     */
+	public void setDebugCounterService(IDebugCounterService debugCounter)
+			throws CounterException;
+
+	/**
+     * Start this switch driver's sub handshake. This might be a no-op but
+     * this method must be called at least once for the switch to be become
+     * ready.
+     * This method must only be called from the I/O thread
+	 * @throws IOException
+     * @throws SwitchDriverSubHandshakeAlreadyStarted if the sub-handshake has
+     * already been started
+     */
+    public void startDriverHandshake() throws IOException;
+
+    /**
+     * Check if the sub-handshake for this switch driver has been completed.
+     * This method can only be called after startDriverHandshake()
+     *
+     * This methods must only be called from the I/O thread
+     * @return true if the sub-handshake has been completed. False otherwise
+     * @throws SwitchDriverSubHandshakeNotStarted if startDriverHandshake() has
+     * not been called yet.
+     */
+    public boolean isDriverHandshakeComplete();
+
+    /**
+     * Pass the given OFMessage to the driver as part of this driver's
+     * sub-handshake. Must not be called after the handshake has been completed
+     * This methods must only be called from the I/O thread
+     * @param m The message that the driver should process
+     * @throws SwitchDriverSubHandshakeCompleted if isDriverHandshake() returns
+     * false before this method call
+     * @throws SwitchDriverSubHandshakeNotStarted if startDriverHandshake() has
+     * not been called yet.
+     */
+    public void processDriverHandshakeMessage(OFMessage m);
+
+    /**
+     * Set the flow table full flag in the switch
+     * XXX S Rethink this for multiple tables
+     */
+    public void setTableFull(boolean isFull);
+
 }
diff --git a/src/main/java/net/floodlightcontroller/core/IOFSwitchListener.java b/src/main/java/net/floodlightcontroller/core/IOFSwitchListener.java
index d874d75..caeb95f 100644
--- a/src/main/java/net/floodlightcontroller/core/IOFSwitchListener.java
+++ b/src/main/java/net/floodlightcontroller/core/IOFSwitchListener.java
@@ -17,36 +17,98 @@
 
 package net.floodlightcontroller.core;
 
+import net.floodlightcontroller.core.IOFSwitch.PortChangeType;
+
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+
 /**
- * @author David Erickson (daviderickson@cs.stanford.edu)
+ *
+ * Switch lifecycle notifications.
+ *
+ * These updates /happen-after/ the corresponding changes have been
+ * committed. I.e., the changes are visible when
+ * {@link IFloodlightProviderService#getSwitch(long)}
+ * {@link IFloodlightProviderService#getAllSwitchDpids()}
+ * {@link IFloodlightProviderService#getAllSwitchMap()}
+ * or any method on the IOFSwitch returned by these methods are
+ * called from the notification method or after it.
+ *
+ * Note however, that additional changes could have been committed before
+ * the notification for which the notification is still pending. E.g.,
+ * {@link IFloodlightProviderService#getSwitch(long)} might return null after
+ * a switchAdded() (which happens if the switch has been added and then
+ * removed and the remove hasn't been dispatched yet).
+ *
+ * These lifecycle notification methods are called by a single thread and they
+ * will always be called by the same thread.
+ * The calls are always in order.
+ *
  */
 public interface IOFSwitchListener {
 
     /**
-     * Fired when a switch is connected to the controller, and has sent
-     * a features reply.
-     *
-     * @param sw
-     */
-    public void addedSwitch(IOFSwitch sw);
-
-    /**
-     * Fired when a switch is disconnected from the controller.
-     *
-     * @param sw
-     */
-    public void removedSwitch(IOFSwitch sw);
-
-    /**
-     * Fired when ports on a switch change (any change to the collection
-     * of OFPhysicalPorts and/or to a particular port)
-     */
-    public void switchPortChanged(Long switchId);
-
-    /**
      * The name assigned to this listener
-     *
-     * @return the name of the listener
      */
     public String getName();
+
+	/**
+     * Fired when switch becomes active on the controller cluster and this
+     * controller instance has acquired MASTER role for this switch
+     * @param switchId the datapath Id of the new switch
+     */
+	public void switchActivatedMaster(long swId);
+
+	/**
+     * Fired when switch becomes active on the controller cluster and this
+     * controller instance has acquired EQUAL role for this switch
+     * @param switchId the datapath Id of the new switch
+     */
+	public void switchActivatedEqual(long swId);
+
+	/**
+     * Fired when the role of this controller for this switch, transitions
+     * from MASTER role to EQUAL role
+     * @param switchId the datapath Id of the new switch
+     */
+	public void switchMasterToEqual(long swId);
+
+	/**
+     * Fired when the role of this controller for this switch, transitions
+     * from EQUAL role to MASTER role
+     * @param switchId the datapath Id of the new switch
+     */
+	public void switchEqualToMaster(long swId);
+
+	/**
+	 * Fired when this switch has disconnected at this controller. It does NOT
+	 * imply that the switch has died, or it has disconnected from any or every
+	 * other controller it was connected to.
+	 * @param swId
+	 */
+	public void switchDisconnected(long swId);
+
+	/**
+     * Fired when a port on a known switch changes.
+     *
+     * A user of this notification needs to take care if the port and type
+     * information is used directly and if the collection of ports has been
+     * queried as well. This notification will only be dispatched after the
+     * the port changes have been committed to the IOFSwitch instance. However,
+     * if a user has previously called {@link IOFSwitch#getPorts()} or related
+     * method a subsequent update might already be present in the information
+     * returned by getPorts.
+     * @param switchId
+     * @param port
+     * @param type
+     */
+	public void switchPortChanged(long swId, OFPortDesc port,
+			PortChangeType changeType);
+
+	/**
+     * Fired when any non-port related information (e.g., attributes,
+     * features) change after a switchAdded XXX S needs more specific methods
+     * TODO: currently unused
+     * @param switchId
+     */
+	//public void switchChanged(long swId);
 }
diff --git a/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeAlreadyStarted.java b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeAlreadyStarted.java
new file mode 100644
index 0000000..eb5541b
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeAlreadyStarted.java
@@ -0,0 +1,15 @@
+package net.floodlightcontroller.core;
+
+/**
+ * Thrown when IOFSwitch.startDriverHandshake() is called more than once.
+ * @author gregor
+ *
+ */
+public class SwitchDriverSubHandshakeAlreadyStarted extends
+    SwitchDriverSubHandshakeException {
+    private static final long serialVersionUID = -5491845708752443501L;
+
+    public SwitchDriverSubHandshakeAlreadyStarted() {
+        super();
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeCompleted.java b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeCompleted.java
new file mode 100644
index 0000000..083eead
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeCompleted.java
@@ -0,0 +1,20 @@
+package net.floodlightcontroller.core;
+
+import org.projectfloodlight.openflow.protocol.OFMessage;
+
+
+/**
+ * Indicates that a message was passed to a switch driver's subhandshake
+ * handling code but the driver has already completed the sub-handshake
+ * @author gregor
+ *
+ */
+public class SwitchDriverSubHandshakeCompleted
+        extends SwitchDriverSubHandshakeException {
+    private static final long serialVersionUID = -8817822245846375995L;
+
+    public SwitchDriverSubHandshakeCompleted(OFMessage m) {
+        super("Sub-Handshake is already complete but received message " +
+              m.getType());
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeException.java b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeException.java
new file mode 100644
index 0000000..0c0c873
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeException.java
@@ -0,0 +1,27 @@
+package net.floodlightcontroller.core;
+
+/**
+ * Base class for exception thrown by switch driver sub-handshake processing
+ * @author gregor
+ *
+ */
+public class SwitchDriverSubHandshakeException extends RuntimeException {
+    private static final long serialVersionUID = -6257836781419604438L;
+
+    protected SwitchDriverSubHandshakeException() {
+        super();
+    }
+
+    protected SwitchDriverSubHandshakeException(String arg0, Throwable arg1) {
+        super(arg0, arg1);
+    }
+
+    protected SwitchDriverSubHandshakeException(String arg0) {
+        super(arg0);
+    }
+
+    protected SwitchDriverSubHandshakeException(Throwable arg0) {
+        super(arg0);
+    }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeNotStarted.java b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeNotStarted.java
new file mode 100644
index 0000000..67ec68b
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeNotStarted.java
@@ -0,0 +1,16 @@
+package net.floodlightcontroller.core;
+
+/**
+ * Thrown when a switch driver's sub-handshake has not been started but an
+ * operation requiring the sub-handshake has been attempted.
+ * @author gregor
+ *
+ */
+public class SwitchDriverSubHandshakeNotStarted extends
+    SwitchDriverSubHandshakeException {
+    private static final long serialVersionUID = -5491845708752443501L;
+
+    public SwitchDriverSubHandshakeNotStarted() {
+        super();
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeStateException.java b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeStateException.java
new file mode 100644
index 0000000..1f49aea
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/core/SwitchDriverSubHandshakeStateException.java
@@ -0,0 +1,16 @@
+package net.floodlightcontroller.core;
+
+/**
+ * Thrown when a switch driver's sub-handshake state-machine receives an
+ * unexpected OFMessage and/or is in an invald state
+ * @author gregor
+ *
+ */
+public class SwitchDriverSubHandshakeStateException extends
+    SwitchDriverSubHandshakeException {
+    private static final long serialVersionUID = -8249926069195147051L;
+
+    public SwitchDriverSubHandshakeStateException(String msg) {
+        super(msg);
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/core/internal/Controller.java b/src/main/java/net/floodlightcontroller/core/internal/Controller.java
index a502711..e80311b 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/Controller.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/Controller.java
@@ -19,11 +19,10 @@
 
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
-import java.nio.channels.ClosedChannelException;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -39,30 +38,39 @@
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 
 import net.floodlightcontroller.core.FloodlightContext;
 import net.floodlightcontroller.core.IFloodlightProviderService;
 import net.floodlightcontroller.core.IListener.Command;
 import net.floodlightcontroller.core.IOFMessageListener;
 import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.IOFSwitch.PortChangeType;
 import net.floodlightcontroller.core.IOFSwitchFilter;
 import net.floodlightcontroller.core.IOFSwitchListener;
 import net.floodlightcontroller.core.IUpdate;
 import net.floodlightcontroller.core.annotations.LogMessageDoc;
 import net.floodlightcontroller.core.annotations.LogMessageDocs;
-import net.floodlightcontroller.core.internal.OFChannelState.HandshakeState;
+import net.floodlightcontroller.core.internal.OFChannelHandler.RoleRecvStatus;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
 import net.floodlightcontroller.core.util.ListenerDispatcher;
 import net.floodlightcontroller.core.web.CoreWebRoutable;
+import net.floodlightcontroller.debugcounter.IDebugCounter;
+import net.floodlightcontroller.debugcounter.IDebugCounterService;
+import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterException;
+import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType;
+import net.floodlightcontroller.debugevent.IDebugEventService;
+import net.floodlightcontroller.debugevent.IDebugEventService.EventColumn;
+import net.floodlightcontroller.debugevent.IDebugEventService.EventFieldType;
+import net.floodlightcontroller.debugevent.IDebugEventService.EventType;
+import net.floodlightcontroller.debugevent.IDebugEventService.MaxEventsRegistered;
+import net.floodlightcontroller.debugevent.IEventUpdater;
+import net.floodlightcontroller.debugevent.NullDebugEvent;
 import net.floodlightcontroller.restserver.IRestApiService;
 import net.floodlightcontroller.threadpool.IThreadPoolService;
-import net.onrc.onos.api.registry.ILocalSwitchMastershipListener;
+import net.floodlightcontroller.util.LoadMonitor;
+import net.onrc.onos.core.drivermanager.DriverManager;
 import net.onrc.onos.core.linkdiscovery.ILinkDiscoveryService;
-import net.onrc.onos.core.main.IOFSwitchPortListener;
 import net.onrc.onos.core.packet.Ethernet;
 import net.onrc.onos.core.registry.IControllerRegistryService;
 import net.onrc.onos.core.registry.IControllerRegistryService.ControlChangeCallback;
@@ -71,123 +79,84 @@
 import net.onrc.onos.core.util.OnosInstanceId;
 
 import org.jboss.netty.bootstrap.ServerBootstrap;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
-import org.jboss.netty.channel.ChannelHandlerContext;
 import org.jboss.netty.channel.ChannelPipelineFactory;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.ChannelUpstreamHandler;
-import org.jboss.netty.channel.Channels;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
 import org.jboss.netty.channel.group.ChannelGroup;
 import org.jboss.netty.channel.group.DefaultChannelGroup;
 import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
-import org.jboss.netty.handler.timeout.IdleStateAwareChannelUpstreamHandler;
-import org.jboss.netty.handler.timeout.IdleStateEvent;
-import org.jboss.netty.handler.timeout.ReadTimeoutException;
-import org.openflow.protocol.OFEchoReply;
-import org.openflow.protocol.OFError;
-import org.openflow.protocol.OFError.OFBadActionCode;
-import org.openflow.protocol.OFError.OFBadRequestCode;
-import org.openflow.protocol.OFError.OFErrorType;
-import org.openflow.protocol.OFError.OFFlowModFailedCode;
-import org.openflow.protocol.OFError.OFHelloFailedCode;
-import org.openflow.protocol.OFError.OFPortModFailedCode;
-import org.openflow.protocol.OFError.OFQueueOpFailedCode;
-import org.openflow.protocol.OFFeaturesReply;
-import org.openflow.protocol.OFGetConfigReply;
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFPacketIn;
-import org.openflow.protocol.OFPhysicalPort;
-import org.openflow.protocol.OFPhysicalPort.OFPortConfig;
-import org.openflow.protocol.OFPhysicalPort.OFPortState;
-import org.openflow.protocol.OFPortStatus;
-import org.openflow.protocol.OFPortStatus.OFPortReason;
-import org.openflow.protocol.OFSetConfig;
-import org.openflow.protocol.OFStatisticsRequest;
-import org.openflow.protocol.OFSwitchConfig;
-import org.openflow.protocol.OFType;
-import org.openflow.protocol.OFVendor;
-import org.openflow.protocol.factory.BasicFactory;
-import org.openflow.protocol.factory.MessageParseException;
-import org.openflow.protocol.statistics.OFDescriptionStatistics;
-import org.openflow.protocol.statistics.OFStatistics;
-import org.openflow.protocol.statistics.OFStatisticsType;
-import org.openflow.protocol.vendor.OFBasicVendorDataType;
-import org.openflow.protocol.vendor.OFBasicVendorId;
-import org.openflow.protocol.vendor.OFVendorId;
-import org.openflow.util.HexString;
-import org.openflow.vendor.nicira.OFNiciraVendorData;
-import org.openflow.vendor.nicira.OFRoleReplyVendorData;
-import org.openflow.vendor.nicira.OFRoleRequestVendorData;
-import org.openflow.vendor.nicira.OFRoleVendorData;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFFactories;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFPacketIn;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.protocol.match.MatchField;
+import org.projectfloodlight.openflow.protocol.match.MatchFields;
+import org.projectfloodlight.openflow.types.EthType;
+import org.projectfloodlight.openflow.types.OFPort;
+import org.projectfloodlight.openflow.util.HexString;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 
 /**
  * The main controller class.  Handles all setup and network listeners
- * <p/>
- * Extensions made by ONOS are:
- * - Detailed Port event: PORTCHANGED -> {PORTCHANGED, PORTADDED, PORTREMOVED}
- * Available as net.onrc.onos.core.main.IOFSwitchPortListener
- * - Distributed ownership control of switch through RegistryService(IControllerRegistryService)
- * - Register ONOS services. (IControllerRegistryService)
- * - Additional DEBUG logs
- * - Try using hostname as controller ID, when ID was not explicitly given.
+ * - Distributed ownership control of switch through IControllerRegistryService
  */
 public class Controller implements IFloodlightProviderService {
 
     protected final static Logger log = LoggerFactory.getLogger(Controller.class);
-
-    private static final String ERROR_DATABASE =
+    static final String ERROR_DATABASE =
             "The controller could not communicate with the system database.";
+    protected static OFFactory factory13 = OFFactories.getFactory(OFVersion.OF_13);
+    protected static OFFactory factory10 = OFFactories.getFactory(OFVersion.OF_10);
 
-    protected BasicFactory factory;
-    protected ConcurrentMap<OFType,
-            ListenerDispatcher<OFType, IOFMessageListener>>
-            messageListeners;
-    // The activeSwitches map contains only those switches that are actively
-    // being controlled by us -- it doesn't contain switches that are
-    // in the slave role
-    protected ConcurrentHashMap<Long, IOFSwitch> activeSwitches;
-    // connectedSwitches contains all connected switches, including ones where
-    // we're a slave controller. We need to keep track of them so that we can
-    // send role request messages to switches when our role changes to master
-    // We add a switch to this set after it successfully completes the
-    // handshake. Access to this Set needs to be synchronized with roleChanger
-    protected HashSet<OFSwitchImpl> connectedSwitches;
+    // connectedSwitches cache contains all connected switch's channelHandlers
+    // including ones where this controller is a master/equal/slave controller
+    // as well as ones that have not been activated yet
+    protected ConcurrentHashMap<Long, OFChannelHandler> connectedSwitches;
+    // These caches contains only those switches that are active
+    protected ConcurrentHashMap<Long, IOFSwitch> activeMasterSwitches;
+    protected ConcurrentHashMap<Long, IOFSwitch> activeEqualSwitches;
+    // lock to synchronize on, when manipulating multiple caches above
+    private Object multiCacheLock;
 
-    // The controllerNodeIPsCache maps Controller IDs to their IP address. 
+    // The controllerNodeIPsCache maps Controller IDs to their IP address.
     // It's only used by handleControllerNodeIPsChanged
     protected HashMap<String, String> controllerNodeIPsCache;
-
-    protected Set<IOFSwitchListener> switchListeners;
     protected BlockingQueue<IUpdate> updates;
 
-    private CopyOnWriteArrayList<ILocalSwitchMastershipListener> localSwitchMastershipListeners =
-        new CopyOnWriteArrayList<>();
+    protected ConcurrentMap<OFType,
+        ListenerDispatcher<OFType, IOFMessageListener>> messageListeners;
+    protected Set<IOFSwitchListener> switchListeners;
 
     // Module dependencies
     protected IRestApiService restApi;
     protected IThreadPoolService threadPool;
     protected IControllerRegistryService registryService;
+    protected IDebugCounterService debugCounters;
+    protected IDebugEventService debugEvents;
 
     protected ILinkDiscoveryService linkDiscovery;
 
     // Configuration options
     protected int openFlowPort = 6633;
     protected int workerThreads = 0;
+
     // The id for this controller node. Should be unique for each controller
     // node in a controller cluster.
     private OnosInstanceId onosInstanceId = new OnosInstanceId("localhost");
 
-    // The current role of the controller.
-    // If the controller isn't configured to support roles, then this is null.
-    protected Role role;
-    // A helper that handles sending and timeout handling for role requests
-    protected RoleChanger roleChanger;
+    // defined counters
+    private Counters counters;
+    // Event IDs for debug events
+    protected IEventUpdater<SwitchEvent> evSwitch;
+
+    // Load monitor for overload protection
+    protected final boolean overload_drop =
+        Boolean.parseBoolean(System.getProperty("overload_drop", "false"));
+    protected final LoadMonitor loadmonitor = new LoadMonitor(log);
 
     // Start time of the controller
     protected long systemStartTime;
@@ -200,12 +169,44 @@
     protected static final int BATCH_MAX_SIZE = 100;
     protected static final boolean ALWAYS_DECODE_ETH = true;
 
+
+
+    // ******************************
+    // Switch Management and Updates
+    // ******************************
+
+    /**
+     * Switch updates are sent to all IOFSwitchListeners. A switch that is connected
+     * to this controller instance, but not activated, is not available for updates.
+     *
+     * In ONOS, each controller instance can simultaneously serve in a MASTER role
+     * for some connected switches, and in a EQUAL role for other connected switches.
+     * The EQUAL role can be treated as a SLAVE role, by ensuring that the
+     * controller instance never sends packets or commands out to the switch.
+     * Activated switches, either with Controller Role MASTER or EQUAL are announced
+     * as updates. We also support announcements of controller role transitions
+     * from MASTER --> EQUAL, and EQUAL --> MASTER, for an individual switch.
+     *
+     * Disconnection of only activated switches are announced. Finally, changes
+     * to switch ports are announced with a portChangeType (see @IOFSwitch)
+     *
+     * @author saurav
+     */
     public enum SwitchUpdateType {
-        ADDED,
-        REMOVED,
+        /** switch activated with this controller's role as MASTER */
+        ACTIVATED_MASTER,
+        /** switch activated with this controller's role as EQUAL.
+         *  listener can treat this controller's role as SLAVE by not
+         *  sending packets or commands to the switch */
+        ACTIVATED_EQUAL,
+        /** this controller's role for this switch changed from Master to Equal */
+        MASTER_TO_EQUAL,
+        /** this controller's role for this switch changed form Equal to Master */
+        EQUAL_TO_MASTER,
+        /** A previously activated switch disconnected */
+        DISCONNECTED,
+        /** Port changed on a previously activated switch */
         PORTCHANGED,
-        PORTADDED,
-        PORTREMOVED
     }
 
     /**
@@ -213,59 +214,269 @@
      * ONOS: This message extended to indicate Port add or removed event.
      */
     protected class SwitchUpdate implements IUpdate {
-        public IOFSwitch sw;
-        public OFPhysicalPort port; // Added by ONOS
-        public SwitchUpdateType switchUpdateType;
-
-        public SwitchUpdate(IOFSwitch sw, SwitchUpdateType switchUpdateType) {
-            this.sw = sw;
-            this.switchUpdateType = switchUpdateType;
+        public long getSwId() {
+            return swId;
         }
 
-        public SwitchUpdate(IOFSwitch sw, OFPhysicalPort port, SwitchUpdateType switchUpdateType) {
-            this.sw = sw;
+        public SwitchUpdateType getSwitchUpdateType() {
+            return switchUpdateType;
+        }
+
+        public PortChangeType getPortChangeType() {
+            return changeType;
+        }
+
+        private final long swId;
+        private final SwitchUpdateType switchUpdateType;
+        private final OFPortDesc port;
+        private final PortChangeType changeType;
+
+        public SwitchUpdate(long swId, SwitchUpdateType switchUpdateType) {
+            this(swId, switchUpdateType, null, null);
+        }
+        public SwitchUpdate(long swId,
+                            SwitchUpdateType switchUpdateType,
+                            OFPortDesc port,
+                            PortChangeType changeType) {
+            if (switchUpdateType == SwitchUpdateType.PORTCHANGED) {
+                if (port == null) {
+                    throw new NullPointerException("Port must not be null " +
+                            "for PORTCHANGED updates");
+                }
+                if (changeType == null) {
+                    throw new NullPointerException("ChangeType must not be " +
+                            "null for PORTCHANGED updates");
+                }
+            } else {
+                if (port != null || changeType != null) {
+                    throw new IllegalArgumentException("port and changeType " +
+                            "must be null for " + switchUpdateType +
+                            " updates");
+                }
+            }
+            this.swId = swId;
+            this.switchUpdateType = switchUpdateType;
             this.port = port;
-            this.switchUpdateType = switchUpdateType;
+            this.changeType = changeType;
         }
 
+        @Override
         public void dispatch() {
             if (log.isTraceEnabled()) {
                 log.trace("Dispatching switch update {} {}",
-                        sw, switchUpdateType);
+                        HexString.toHexString(swId), switchUpdateType);
             }
             if (switchListeners != null) {
                 for (IOFSwitchListener listener : switchListeners) {
-                    switch (switchUpdateType) {
-                        case ADDED:
-                            listener.addedSwitch(sw);
+                    switch(switchUpdateType) {
+                        case ACTIVATED_MASTER:
+                            // don't count here. We have more specific
+                            // counters before the update is created
+                            listener.switchActivatedMaster(swId);
                             break;
-                        case REMOVED:
-                            listener.removedSwitch(sw);
+                        case ACTIVATED_EQUAL:
+                            // don't count here. We have more specific
+                            // counters before the update is created
+                            listener.switchActivatedEqual(swId);
+                            break;
+                        case MASTER_TO_EQUAL:
+                            listener.switchMasterToEqual(swId);
+                            break;
+                        case EQUAL_TO_MASTER:
+                            listener.switchEqualToMaster(swId);
+                            break;
+                        case DISCONNECTED:
+                            // don't count here. We have more specific
+                            // counters before the update is created
+                            listener.switchDisconnected(swId);
                             break;
                         case PORTCHANGED:
-                            listener.switchPortChanged(sw.getId());
-                            break;
-                        case PORTADDED:
-                            if (listener instanceof IOFSwitchPortListener) {
-                                ((IOFSwitchPortListener) listener).switchPortAdded(sw.getId(), port);
-                            }
-                            break;
-                        case PORTREMOVED:
-                            if (listener instanceof IOFSwitchPortListener) {
-                                ((IOFSwitchPortListener) listener).switchPortRemoved(sw.getId(), port);
-                            }
-                            break;
-                        default:
+                            counters.switchPortChanged.updateCounterWithFlush();
+                            listener.switchPortChanged(swId, port, changeType);
                             break;
                     }
                 }
             }
         }
+
+
+    }
+
+    protected boolean addConnectedSwitch(long dpid, OFChannelHandler h) {
+        if (connectedSwitches.get(dpid) != null) {
+            log.error("Trying to add connectedSwitch but found a previous "
+                    + "value for dpid: {}", dpid);
+            return false;
+        } else {
+            connectedSwitches.put(dpid, h);
+            return true;
+        }
+    }
+
+    /**
+     *  Switch Events
+     */
+    @Override
+    public void addSwitchEvent(long dpid, String reason, boolean flushNow) {
+        if (flushNow)
+            evSwitch.updateEventWithFlush(new SwitchEvent(dpid, reason));
+        else
+            evSwitch.updateEventNoFlush(new SwitchEvent(dpid, reason));
+    }
+
+    private boolean validActivation(long dpid) {
+        if (connectedSwitches.get(dpid) == null) {
+            log.error("Trying to activate switch but is not in "
+                    + "connected switches: dpid {}. Aborting ..",
+                    HexString.toHexString(dpid));
+            return false;
+        }
+        if (activeMasterSwitches.get(dpid) != null ||
+                activeEqualSwitches.get(dpid) != null) {
+            log.error("Trying to activate switch but it is already "
+                    + "activated: dpid {}. Found in activeMaster: {} "
+                    + "Found in activeEqual: {}. Aborting ..", new Object[] {
+                            HexString.toHexString(dpid),
+                            (activeMasterSwitches.get(dpid) == null)? 'Y': 'N',
+                            (activeEqualSwitches.get(dpid) == null)? 'Y': 'N'});
+            counters.switchWithSameDpidActivated.updateCounterWithFlush();
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Called when a switch is activated, with this controller's role as MASTER
+     */
+    protected boolean addActivatedMasterSwitch(long dpid, IOFSwitch sw) {
+        synchronized(multiCacheLock) {
+            if (!validActivation(dpid)) return false;
+            activeMasterSwitches.put(dpid, sw);
+        }
+        // XXX Workaround to prevent race condition where a link is detected
+        // and attempted to be written to the database before the port is in
+        // the database. We now suppress link discovery on ports until we're
+        // sure they're in the database.
+        for (OFPortDesc port : sw.getPorts()) {
+            linkDiscovery.disableDiscoveryOnPort(sw.getId(),
+                    port.getPortNo().getShortPortNumber());
+        }
+        //update counters and events
+        counters.switchActivated.updateCounterWithFlush();
+        evSwitch.updateEventWithFlush(new SwitchEvent(dpid, "activeMaster"));
+        addUpdateToQueue(new SwitchUpdate(dpid,
+                SwitchUpdateType.ACTIVATED_MASTER));
+        return true;
+    }
+
+    /**
+     * Called when a switch is activated, with this controller's role as EQUAL
+     */
+    protected boolean addActivatedEqualSwitch(long dpid, IOFSwitch sw) {
+        synchronized(multiCacheLock) {
+            if (!validActivation(dpid)) return false;
+            activeEqualSwitches.put(dpid, sw);
+        }
+        //update counters and events
+        counters.switchActivated.updateCounterWithFlush();
+        evSwitch.updateEventWithFlush(new SwitchEvent(dpid, "activeEqual"));
+        addUpdateToQueue(new SwitchUpdate(dpid,
+                SwitchUpdateType.ACTIVATED_EQUAL));
+        return true;
+    }
+
+    /**
+     * Called when this controller's role for a switch transitions from equal
+     * to master. For 1.0 switches, we internally refer to the role 'slave' as
+     * 'equal' - so this transition is equivalent to 'addActivatedMasterSwitch'.
+     */
+    protected void transitionToMasterSwitch(long dpid) {
+	synchronized(multiCacheLock) {
+		IOFSwitch sw = activeEqualSwitches.remove(dpid);
+		if (sw == null) {
+			log.error("Transition to master called on sw {}, but switch "
+					+ "was not found in controller-cache", dpid);
+			return;
+		}
+		activeMasterSwitches.put(dpid, sw);
+	}
+	addUpdateToQueue(new SwitchUpdate(dpid,
+                SwitchUpdateType.EQUAL_TO_MASTER));
+    }
+
+
+    /**
+     * Called when this controller's role for a switch transitions to equal.
+     * For 1.0 switches, we internally refer to the role 'slave' as
+     * 'equal'.
+     */
+    protected void transitionToEqualSwitch(long dpid) {
+	synchronized(multiCacheLock) {
+		IOFSwitch sw = activeMasterSwitches.remove(dpid);
+		if (sw == null) {
+			log.error("Transition to equal called on sw {}, but switch "
+					+ "was not found in controller-cache", dpid);
+			return;
+		}
+		activeEqualSwitches.put(dpid, sw);
+	}
+	addUpdateToQueue(new SwitchUpdate(dpid,
+                SwitchUpdateType.MASTER_TO_EQUAL));
+    }
+
+    /**
+     * Clear all state in controller switch maps for a switch that has
+     * disconnected from the local controller. Also release control for
+     * that switch from the global repository. Notify switch listeners.
+     */
+    protected void removeConnectedSwitch(long dpid) {
+        releaseRegistryControl(dpid);
+        connectedSwitches.remove(dpid);
+        IOFSwitch sw = activeMasterSwitches.remove(dpid);
+        if (sw == null) sw = activeEqualSwitches.remove(dpid);
+        if (sw != null) {
+            sw.cancelAllStatisticsReplies();
+            sw.setConnected(false); // do we need this?
+        }
+        evSwitch.updateEventWithFlush(new SwitchEvent(dpid, "disconnected"));
+        counters.switchDisconnected.updateCounterWithFlush();
+        addUpdateToQueue(new SwitchUpdate(dpid, SwitchUpdateType.DISCONNECTED));
+    }
+
+    /**
+     * Indicates that ports on the given switch have changed. Enqueue a
+     * switch update.
+     * @param sw
+     */
+    protected void notifyPortChanged(long dpid, OFPortDesc port,
+            PortChangeType changeType) {
+        if (port == null || changeType == null) {
+            String msg = String.format("Switch port or changetType must not "
+                    + "be null in port change notification");
+            throw new NullPointerException(msg);
+        }
+        if (connectedSwitches.get(dpid) == null || getSwitch(dpid) == null) {
+            log.warn("Port change update on switch {} not connected or activated "
+                    + "... Aborting.", HexString.toHexString(dpid));
+            return;
+        }
+
+        if (changeType == PortChangeType.ADD) {
+            // XXX Workaround to prevent race condition where a link is detected
+            // and attempted to be written to the database before the port is in
+            // the database. We now suppress link discovery on ports until we're
+            // sure they're in the database.
+            linkDiscovery.disableDiscoveryOnPort(dpid, port.getPortNo().getShortPortNumber());
+        }
+
+        SwitchUpdate update = new SwitchUpdate(dpid, SwitchUpdateType.PORTCHANGED,
+                port, changeType);
+        addUpdateToQueue(update);
     }
 
     // ***************
     // Getters/Setters
-    // *************** 
+    // ***************
 
     public void setRestApiService(IRestApiService restApi) {
         this.restApi = restApi;
@@ -283,920 +494,120 @@
         this.linkDiscovery = linkDiscovery;
     }
 
-    public void publishUpdate(IUpdate update) {
-        try {
-            this.updates.put(update);
-        } catch (InterruptedException e) {
-            log.error("Failure adding update to queue", e);
-        }
+    public void setDebugCounter(IDebugCounterService debugCounters) {
+        this.debugCounters = debugCounters;
+    }
+
+    public void setDebugEvent(IDebugEventService debugEvents) {
+        this.debugEvents = debugEvents;
+    }
+
+    IDebugCounterService getDebugCounter() {
+        return this.debugCounters;
     }
 
     // **********************
-    // ChannelUpstreamHandler
+    // Role Handling
     // **********************
 
-    /**
-     * Return a new channel handler for processing a switch connections
-     *
-     * @param state The channel state object for the connection
-     * @return the new channel handler
+    /** created by ONOS - works with registry service
      */
-    protected ChannelUpstreamHandler getChannelHandler(OFChannelState state) {
-        return new OFChannelHandler(state);
-    }
-
     protected class RoleChangeCallback implements ControlChangeCallback {
         @Override
         public void controlChanged(long dpidLong, boolean hasControl) {
             Dpid dpid = new Dpid(dpidLong);
             log.info("Role change callback for switch {}, hasControl {}",
-                     dpid, hasControl);
+                    dpid, hasControl);
 
-            synchronized (roleChanger) {
-                Role role = null;
-                /*
-                 * issue #229
-                 * Cannot rely on sw.getRole() as it can be behind due to pending
-                 * role changes in the queue. Just submit it and late the RoleChanger
-                 * handle duplicates.
-                 */
-                if (hasControl) {
-                    role = Role.MASTER;
-                } else {
-                    role = Role.SLAVE;
-                }
-                //
-                // Inform all listeners about the Controller role change.
-                //
-                // NOTE: The role change here is triggered by the mastership
-                // election mechanism, and reflects what the Controller itself
-                // believes its role should be. The role change request hasn't
-                // been accepted by the switch itself.
-                // If the semantics for informing the listeners is changed
-                // (i.e., the listeners should be informed after the switch
-                // has accepted the role change), then the code below should
-                // be moved to method handleRoleReplyMessage() or its
-                // equivalent.
-                // 
-                for (ILocalSwitchMastershipListener listener : localSwitchMastershipListeners) {
-                    listener.controllerRoleChanged(dpid, role);
-                }
-
-                OFSwitchImpl sw = null;
-                for (OFSwitchImpl connectedSw : connectedSwitches) {
-                    if (connectedSw.getId() == dpidLong) {
-                        sw = connectedSw;
-                        break;
-                    }
-                }
-                if (sw == null) {
-                    log.warn("Switch {} not found in connected switches",
-                             dpid);
-                    return;
-                }
-
-                log.debug("Sending role request {} msg to {}", role, sw);
-                Collection<OFSwitchImpl> swList = new ArrayList<OFSwitchImpl>(1);
-                swList.add(sw);
-                roleChanger.submitRequest(swList, role);
-            }
-        }
-    }
-
-    /**
-     * Channel handler deals with the switch connection and dispatches
-     * switch messages to the appropriate locations.
-     *
-     * @author readams
-     */
-    protected class OFChannelHandler
-            extends IdleStateAwareChannelUpstreamHandler {
-        protected OFSwitchImpl sw;
-        protected OFChannelState state;
-
-        public OFChannelHandler(OFChannelState state) {
-            this.state = state;
-        }
-
-        @Override
-        @LogMessageDoc(message = "New switch connection from {ip address}",
-                explanation = "A new switch has connected from the " +
-                        "specified IP address")
-        public void channelConnected(ChannelHandlerContext ctx,
-                                     ChannelStateEvent e) throws Exception {
-            log.info("New switch connection from {}",
-                    e.getChannel().getRemoteAddress());
-
-            sw = new OFSwitchImpl();
-            sw.setChannel(e.getChannel());
-            sw.setFloodlightProvider(Controller.this);
-            sw.setThreadPoolService(threadPool);
-
-            List<OFMessage> msglist = new ArrayList<OFMessage>(1);
-            msglist.add(factory.getMessage(OFType.HELLO));
-            e.getChannel().write(msglist);
-
-        }
-
-        @Override
-        @LogMessageDoc(message = "Disconnected switch {switch information}",
-                explanation = "The specified switch has disconnected.")
-        public void channelDisconnected(ChannelHandlerContext ctx,
-                                        ChannelStateEvent e) throws Exception {
-            if (sw != null && state.hsState == HandshakeState.READY) {
-                if (activeSwitches.containsKey(sw.getId())) {
-                    // It's safe to call removeSwitch even though the map might
-                    // not contain this particular switch but another with the 
-                    // same DPID
-                    removeSwitch(sw);
-                }
-                synchronized (roleChanger) {
-                    if (controlRequested) {
-
-                        //
-                        // Inform all listeners about the switch disconnection
-                        //
-                        Dpid dpid = new Dpid(sw.getId());
-                        for (ILocalSwitchMastershipListener listener :
-                                 localSwitchMastershipListeners) {
-                            listener.switchDisconnected(dpid);
-                        }
-
-                        registryService.releaseControl(sw.getId());
-                    }
-                    connectedSwitches.remove(sw);
-                }
-                sw.setConnected(false);
-            }
-            log.info("Disconnected switch {}", sw);
-        }
-
-        @Override
-        @LogMessageDocs({
-                @LogMessageDoc(level = "ERROR",
-                        message = "Disconnecting switch {switch} due to read timeout",
-                        explanation = "The connected switch has failed to send any " +
-                                "messages or respond to echo requests",
-                        recommendation = LogMessageDoc.CHECK_SWITCH),
-                @LogMessageDoc(level = "ERROR",
-                        message = "Disconnecting switch {switch}: failed to " +
-                                "complete handshake",
-                        explanation = "The switch did not respond correctly " +
-                                "to handshake messages",
-                        recommendation = LogMessageDoc.CHECK_SWITCH),
-                @LogMessageDoc(level = "ERROR",
-                        message = "Disconnecting switch {switch} due to IO Error: {}",
-                        explanation = "There was an error communicating with the switch",
-                        recommendation = LogMessageDoc.CHECK_SWITCH),
-                @LogMessageDoc(level = "ERROR",
-                        message = "Disconnecting switch {switch} due to switch " +
-                                "state error: {error}",
-                        explanation = "The switch sent an unexpected message",
-                        recommendation = LogMessageDoc.CHECK_SWITCH),
-                @LogMessageDoc(level = "ERROR",
-                        message = "Disconnecting switch {switch} due to " +
-                                "message parse failure",
-                        explanation = "Could not parse a message from the switch",
-                        recommendation = LogMessageDoc.CHECK_SWITCH),
-                @LogMessageDoc(level = "ERROR",
-                        message = "Could not process message: queue full",
-                        explanation = "OpenFlow messages are arriving faster than " +
-                                " the controller can process them.",
-                        recommendation = LogMessageDoc.CHECK_CONTROLLER),
-                @LogMessageDoc(level = "ERROR",
-                        message = "Error while processing message " +
-                                "from switch {switch} {cause}",
-                        explanation = "An error occurred processing the switch message",
-                        recommendation = LogMessageDoc.GENERIC_ACTION)
-        })
-        public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e)
-                throws Exception {
-            if (e.getCause() instanceof ReadTimeoutException) {
-                // switch timeout
-                log.error("Disconnecting switch {} due to read timeout", sw);
-                ctx.getChannel().close();
-            } else if (e.getCause() instanceof HandshakeTimeoutException) {
-                log.error("Disconnecting switch {}: failed to complete handshake",
-                        sw);
-                ctx.getChannel().close();
-            } else if (e.getCause() instanceof ClosedChannelException) {
-                //log.warn("Channel for sw {} already closed", sw);
-            } else if (e.getCause() instanceof IOException) {
-                log.error("Disconnecting switch {} due to IO Error: {}",
-                        sw, e.getCause().getMessage());
-                ctx.getChannel().close();
-            } else if (e.getCause() instanceof SwitchStateException) {
-                log.error("Disconnecting switch {} due to switch state error: {}",
-                        sw, e.getCause().getMessage());
-                ctx.getChannel().close();
-            } else if (e.getCause() instanceof MessageParseException) {
-                log.error("Disconnecting switch " + sw +
-                        " due to message parse failure",
-                        e.getCause());
-                ctx.getChannel().close();
-            } else if (e.getCause() instanceof RejectedExecutionException) {
-                log.warn("Could not process message: queue full");
-            } else {
-                log.error("Error while processing message from switch " + sw,
-                        e.getCause());
-                ctx.getChannel().close();
-            }
-        }
-
-        @Override
-        public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e)
-                throws Exception {
-            List<OFMessage> msglist = new ArrayList<OFMessage>(1);
-            msglist.add(factory.getMessage(OFType.ECHO_REQUEST));
-            e.getChannel().write(msglist);
-        }
-
-        @Override
-        public void messageReceived(ChannelHandlerContext ctx, MessageEvent e)
-                throws Exception {
-            if (e.getMessage() instanceof List) {
-                @SuppressWarnings("unchecked")
-                List<OFMessage> msglist = (List<OFMessage>) e.getMessage();
-
-                for (OFMessage ofm : msglist) {
-                    try {
-                        processOFMessage(ofm);
-                    } catch (Exception ex) {
-                        // We are the last handler in the stream, so run the 
-                        // exception through the channel again by passing in 
-                        // ctx.getChannel().
-                        Channels.fireExceptionCaught(ctx.getChannel(), ex);
-                    }
-                }
-
-                // Flush all flow-mods/packet-out generated from this "train"
-                OFSwitchImpl.flush_all();
-            }
-        }
-
-        /**
-         * Process the request for the switch description
-         */
-        @LogMessageDoc(level = "ERROR",
-                message = "Exception in reading description " +
-                        " during handshake {exception}",
-                explanation = "Could not process the switch description string",
-                recommendation = LogMessageDoc.CHECK_SWITCH)
-        @SuppressWarnings("unchecked")
-        void processSwitchDescReply() {
-            try {
-                // Read description, if it has been updated
-                Future<?> future =
-                        (Future<?>) sw.
-                                getAttribute(IOFSwitch.SWITCH_DESCRIPTION_FUTURE);
-                Future<List<OFStatistics>> desc_future =
-                        (Future<List<OFStatistics>>) future;
-                List<OFStatistics> values =
-                        desc_future.get(0, TimeUnit.MILLISECONDS);
-                if (values != null) {
-                    OFDescriptionStatistics description =
-                            new OFDescriptionStatistics();
-                    ChannelBuffer data =
-                            ChannelBuffers.buffer(description.getLength());
-                    for (OFStatistics f : values) {
-                        f.writeTo(data);
-                        description.readFrom(data);
-                        break; // SHOULD be a list of length 1
-                    }
-                    sw.setAttribute(IOFSwitch.SWITCH_DESCRIPTION_DATA,
-                            description);
-                    sw.setSwitchProperties(description);
-                    data = null;
-                }
-
-                sw.removeAttribute(IOFSwitch.SWITCH_DESCRIPTION_FUTURE);
-                state.hasDescription = true;
-                checkSwitchReady();
-            } catch (InterruptedException ex) {
-                // Ignore
-            } catch (TimeoutException ex) {
-                // Ignore
-            } catch (Exception ex) {
-                log.error("Exception in reading description " +
-                        " during handshake", ex);
-            }
-        }
-
-        /**
-         * Send initial switch setup information that we need before adding
-         * the switch
-         *
-         * @throws IOException
-         */
-        void sendHelloConfiguration() throws IOException {
-            // Send initial Features Request
-            log.debug("Sending FEATURES_REQUEST to {}", sw);
-            sw.write(factory.getMessage(OFType.FEATURES_REQUEST), null);
-        }
-
-        /**
-         * Send the configuration requests we can only do after we have
-         * the features reply
-         *
-         * @throws IOException
-         */
-        void sendFeatureReplyConfiguration() throws IOException {
-            log.debug("Sending CONFIG_REQUEST to {}", sw);
-            // Ensure we receive the full packet via PacketIn
-            OFSetConfig config = (OFSetConfig) factory
-                    .getMessage(OFType.SET_CONFIG);
-            config.setMissSendLength((short) 0xffff)
-                    .setLengthU(OFSwitchConfig.MINIMUM_LENGTH);
-            sw.write(config, null);
-            sw.write(factory.getMessage(OFType.GET_CONFIG_REQUEST),
-                    null);
-
-            // Get Description to set switch-specific flags
-            OFStatisticsRequest req = new OFStatisticsRequest();
-            req.setStatisticType(OFStatisticsType.DESC);
-            req.setLengthU(req.getLengthU());
-            Future<List<OFStatistics>> dfuture =
-                    sw.getStatistics(req);
-            sw.setAttribute(IOFSwitch.SWITCH_DESCRIPTION_FUTURE,
-                    dfuture);
-
-        }
-
-        volatile Boolean controlRequested = Boolean.FALSE;
-
-        protected void checkSwitchReady() {
-            if (state.hsState == HandshakeState.FEATURES_REPLY &&
-                    state.hasDescription && state.hasGetConfigReply) {
-
-                state.hsState = HandshakeState.READY;
-                log.debug("Handshake with {} complete", sw);
-
-                synchronized (roleChanger) {
-                    // We need to keep track of all of the switches that are connected
-                    // to the controller, in any role, so that we can later send the
-                    // role request messages when the controller role changes.
-                    // We need to be synchronized while doing this: we must not
-                    // send a another role request to the connectedSwitches until
-                    // we were able to add this new switch to connectedSwitches
-                    // *and* send the current role to the new switch.
-                    connectedSwitches.add(sw);
-
-                    if (role != null) {
-                        //Put the switch in SLAVE mode until we know we have control
-                        log.debug("Setting new switch {} to SLAVE", sw.getStringId());
-                        Collection<OFSwitchImpl> swList = new ArrayList<OFSwitchImpl>(1);
-                        swList.add(sw);
-                        roleChanger.submitRequest(swList, Role.SLAVE);
-
-                        //Request control of the switch from the global registry
-                        try {
-                            controlRequested = Boolean.TRUE;
-                            registryService.requestControl(sw.getId(),
-                                    new RoleChangeCallback());
-                        } catch (RegistryException e) {
-                            log.debug("Registry error: {}", e.getMessage());
-                            controlRequested = Boolean.FALSE;
-                        }
-
-
-                        // Send a role request if role support is enabled for the controller
-                        // This is a probe that we'll use to determine if the switch
-                        // actually supports the role request message. If it does we'll
-                        // get back a role reply message. If it doesn't we'll get back an
-                        // OFError message.
-                        // If role is MASTER we will promote switch to active
-                        // list when we receive the switch's role reply messages
-                        /*
-                        log.debug("This controller's role is {}, " +
-                                "sending initial role request msg to {}",
-                                role, sw);
-                        Collection<OFSwitchImpl> swList = new ArrayList<OFSwitchImpl>(1);
-                        swList.add(sw);
-                        roleChanger.submitRequest(swList, role);
-                        */
-                    } else {
-                        // Role supported not enabled on controller (for now)
-                        // automatically promote switch to active state.
-                        log.debug("This controller's role is {}, " +
-                                "not sending role request msg to {}",
-                                role, sw);
-                        // Need to clear FlowMods before we add the switch
-                        // and dispatch updates otherwise we have a race condition.
-                        sw.clearAllFlowMods();
-                        addSwitch(sw);
-                        state.firstRoleReplyReceived = true;
-                    }
-                }
-                if (!controlRequested) {
-                    // yield to allow other thread(s) to release control
-                    try {
-                        Thread.sleep(10);
-                    } catch (InterruptedException e) {
-                        // Ignore interruptions
-                    }
-                    // safer to bounce the switch to reconnect here than proceeding further
-                    log.debug("Closing {} because we weren't able to request control " +
-                            "successfully" + sw);
-                    sw.channel.close();
-                }
-            }
-        }
-
-        /* Handle a role reply message we received from the switch. Since
-         * netty serializes message dispatch we don't need to synchronize
-         * against other receive operations from the same switch, so no need
-         * to synchronize addSwitch(), removeSwitch() operations from the same
-         * connection.
-         * FIXME: However, when a switch with the same DPID connects we do
-         * need some synchronization. However, handling switches with same
-         * DPID needs to be revisited anyways (get rid of r/w-lock and synchronous
-         * removedSwitch notification):1
-         *
-         */
-        @LogMessageDoc(level = "ERROR",
-                message = "Invalid role value in role reply message",
-                explanation = "Was unable to set the HA role (master or slave) " +
-                        "for the controller.",
-                recommendation = LogMessageDoc.CHECK_CONTROLLER)
-        protected void handleRoleReplyMessage(OFVendor vendorMessage,
-                                              OFRoleReplyVendorData roleReplyVendorData) {
-            // Map from the role code in the message to our role enum
-            int nxRole = roleReplyVendorData.getRole();
             Role role = null;
-            switch (nxRole) {
-                case OFRoleVendorData.NX_ROLE_OTHER:
-                    role = Role.EQUAL;
-                    break;
-                case OFRoleVendorData.NX_ROLE_MASTER:
-                    role = Role.MASTER;
-                    break;
-                case OFRoleVendorData.NX_ROLE_SLAVE:
-                    role = Role.SLAVE;
-                    break;
-                default:
-                    log.error("Invalid role value in role reply message");
-                    sw.getChannel().close();
-                    return;
-            }
 
-            log.debug("Handling role reply for role {} from {}. " +
-                    "Controller's role is {} ",
-                    new Object[]{role, sw, Controller.this.role}
-            );
+            /*
+             * issue #229
+             * Cannot rely on sw.getRole() as it can be behind due to pending
+             * role changes in the queue. Just submit it and late the
+             * RoleChanger handle duplicates.
+             */
 
-            sw.deliverRoleReply(vendorMessage.getXid(), role);
-
-            boolean isActive = activeSwitches.containsKey(sw.getId());
-            if (!isActive && sw.isActive()) {
-                // Transition from SLAVE to MASTER.
-
-                if (!state.firstRoleReplyReceived ||
-                        getAlwaysClearFlowsOnSwAdd()) {
-                    // This is the first role-reply message we receive from
-                    // this switch or roles were disabled when the switch
-                    // connected:
-                    // Delete all pre-existing flows for new connections to
-                    // the master
-                    //
-                    // FIXME: Need to think more about what the test should
-                    // be for when we flush the flow-table? For example,
-                    // if all the controllers are temporarily in the backup
-                    // role (e.g. right after a failure of the master
-                    // controller) at the point the switch connects, then
-                    // all of the controllers will initially connect as
-                    // backup controllers and not flush the flow-table.
-                    // Then when one of them is promoted to master following
-                    // the master controller election the flow-table
-                    // will still not be flushed because that's treated as
-                    // a failover event where we don't want to flush the
-                    // flow-table. The end result would be that the flow
-                    // table for a newly connected switch is never
-                    // flushed. Not sure how to handle that case though...
-                    sw.clearAllFlowMods();
-                    log.debug("First role reply from master switch {}, " +
-                            "clear FlowTable to active switch list",
-                            HexString.toHexString(sw.getId()));
-                }
-
-                // Some switches don't seem to update us with port
-                // status messages while in slave role.
-                //readSwitchPortStateFromStorage(sw);
-
-                // Only add the switch to the active switch list if
-                // we're not in the slave role. Note that if the role
-                // attribute is null, then that means that the switch
-                // doesn't support the role request messages, so in that
-                // case we're effectively in the EQUAL role and the
-                // switch should be included in the active switch list.
-                addSwitch(sw);
-                log.debug("Added master switch {} to active switch list",
-                        HexString.toHexString(sw.getId()));
-
-            } else if (isActive && !sw.isActive()) {
-                // Transition from MASTER to SLAVE: remove switch
-                // from active switch list.
-                log.debug("Removed slave switch {} from active switch" +
-                        " list", HexString.toHexString(sw.getId()));
-                removeSwitch(sw);
-            }
-
-            // Indicate that we have received a role reply message.
-            state.firstRoleReplyReceived = true;
-        }
-
-        protected boolean handleVendorMessage(OFVendor vendorMessage) {
-            boolean shouldHandleMessage = false;
-            int vendor = vendorMessage.getVendor();
-            switch (vendor) {
-                case OFNiciraVendorData.NX_VENDOR_ID:
-                    OFNiciraVendorData niciraVendorData =
-                            (OFNiciraVendorData) vendorMessage.getVendorData();
-                    int dataType = niciraVendorData.getDataType();
-                    switch (dataType) {
-                        case OFRoleReplyVendorData.NXT_ROLE_REPLY:
-                            OFRoleReplyVendorData roleReplyVendorData =
-                                    (OFRoleReplyVendorData) niciraVendorData;
-                            handleRoleReplyMessage(vendorMessage,
-                                    roleReplyVendorData);
-                            break;
-                        default:
-                            log.warn("Unhandled Nicira VENDOR message; " +
-                                    "data type = {}", dataType);
-                            break;
-                    }
-                    break;
-                default:
-                    log.warn("Unhandled VENDOR message; vendor id = {}", vendor);
-                    break;
-            }
-
-            return shouldHandleMessage;
-        }
-
-        /**
-         * Dispatch an Openflow message from a switch to the appropriate
-         * handler.
-         *
-         * @param m The message to process
-         * @throws IOException
-         * @throws SwitchStateException
-         */
-        @LogMessageDocs({
-                @LogMessageDoc(level = "WARN",
-                        message = "Config Reply from {switch} has " +
-                                "miss length set to {length}",
-                        explanation = "The controller requires that the switch " +
-                                "use a miss length of 0xffff for correct " +
-                                "function",
-                        recommendation = "Use a different switch to ensure " +
-                                "correct function"),
-                @LogMessageDoc(level = "WARN",
-                        message = "Received ERROR from sw {switch} that "
-                                + "indicates roles are not supported "
-                                + "but we have received a valid "
-                                + "role reply earlier",
-                        explanation = "The switch sent a confusing message to the" +
-                                "controller")
-        })
-        protected void processOFMessage(OFMessage m)
-                throws IOException, SwitchStateException {
-            boolean shouldHandleMessage = false;
-
-            switch (m.getType()) {
-                case HELLO:
-                    if (log.isTraceEnabled())
-                        log.trace("HELLO from {}", sw);
-
-                    if (state.hsState.equals(HandshakeState.START)) {
-                        state.hsState = HandshakeState.HELLO;
-                        sendHelloConfiguration();
-                    } else {
-                        throw new SwitchStateException("Unexpected HELLO from "
-                                + sw);
-                    }
-                    break;
-                case ECHO_REQUEST:
-                    OFEchoReply reply =
-                            (OFEchoReply) factory.getMessage(OFType.ECHO_REPLY);
-                    reply.setXid(m.getXid());
-                    sw.write(reply, null);
-                    break;
-                case ECHO_REPLY:
-                    break;
-                case FEATURES_REPLY:
-                    if (log.isTraceEnabled())
-                        log.trace("Features Reply from {}", sw);
-
-                    sw.setFeaturesReply((OFFeaturesReply) m);
-                    if (state.hsState.equals(HandshakeState.HELLO)) {
-                        sendFeatureReplyConfiguration();
-                        state.hsState = HandshakeState.FEATURES_REPLY;
-                        // uncomment to enable "dumb" switches like cbench
-                        // state.hsState = HandshakeState.READY;
-                        // addSwitch(sw);
-                    } else {
-                        // return results to rest api caller
-                        sw.deliverOFFeaturesReply(m);
-                        // update database */
-                        //updateActiveSwitchInfo(sw);
-                    }
-                    break;
-                case GET_CONFIG_REPLY:
-                    if (log.isTraceEnabled())
-                        log.trace("Get config reply from {}", sw);
-
-                    if (!state.hsState.equals(HandshakeState.FEATURES_REPLY)) {
-                        String em = "Unexpected GET_CONFIG_REPLY from " + sw;
-                        throw new SwitchStateException(em);
-                    }
-                    OFGetConfigReply cr = (OFGetConfigReply) m;
-                    if (cr.getMissSendLength() == (short) 0xffff) {
-                        log.trace("Config Reply from {} confirms " +
-                                "miss length set to 0xffff", sw);
-                    } else {
-                        log.warn("Config Reply from {} has " +
-                                "miss length set to {}",
-                                sw, cr.getMissSendLength() & 0xffff);
-                    }
-                    state.hasGetConfigReply = true;
-                    checkSwitchReady();
-                    break;
-                case VENDOR:
-                    shouldHandleMessage = handleVendorMessage((OFVendor) m);
-                    break;
-                case ERROR:
-                    log.debug("Recieved ERROR message from switch {}: {}", sw, m);
-                    // TODO: we need better error handling. Especially for
-                    // request/reply style message (stats, roles) we should have
-                    // a unified way to lookup the xid in the error message.
-                    // This will probable involve rewriting the way we handle
-                    // request/reply style messages.
-                    OFError error = (OFError) m;
-                    boolean shouldLogError = true;
-                    // TODO: should we check that firstRoleReplyReceived is false,
-                    // i.e., check only whether the first request fails?
-                    if (sw.checkFirstPendingRoleRequestXid(error.getXid())) {
-                        boolean isBadVendorError =
-                                (error.getErrorType() == OFError.OFErrorType.
-                                        OFPET_BAD_REQUEST.getValue());
-                        // We expect to receive a bad vendor error when
-                        // we're connected to a switch that doesn't support
-                        // the Nicira vendor extensions (i.e. not OVS or
-                        // derived from OVS).  By protocol, it should also be
-                        // BAD_VENDOR, but too many switch implementations
-                        // get it wrong and we can already check the xid()
-                        // so we can ignore the type with confidence that this
-                        // is not a spurious error
-                        shouldLogError = !isBadVendorError;
-                        if (isBadVendorError) {
-                            log.debug("Handling bad vendor error for {}", sw);
-                            if (state.firstRoleReplyReceived && (role != null)) {
-                                log.warn("Received ERROR from sw {} that "
-                                        + "indicates roles are not supported "
-                                        + "but we have received a valid "
-                                        + "role reply earlier", sw);
-                            }
-                            state.firstRoleReplyReceived = true;
-                            Role requestedRole =
-                                    sw.deliverRoleRequestNotSupportedEx(error.getXid());
-                            synchronized (roleChanger) {
-                                if (sw.role == null && Controller.this.role == Role.SLAVE) {
-                                    //This will now never happen. The Controller's role
-                                    //is now never SLAVE, always MASTER.
-                                    // the switch doesn't understand role request
-                                    // messages and the current controller role is
-                                    // slave. We need to disconnect the switch.
-                                    // @see RoleChanger for rationale
-                                    log.warn("Closing {} channel because controller's role " +
-                                            "is SLAVE", sw);
-                                    sw.getChannel().close();
-                                } else if (sw.role == null && requestedRole == Role.MASTER) {
-                                    log.debug("Adding switch {} because we got an error" +
-                                            " returned from a MASTER role request", sw);
-                                    // Controller's role is master: add to
-                                    // active
-                                    // TODO: check if clearing flow table is
-                                    // right choice here.
-                                    // Need to clear FlowMods before we add the switch
-                                    // and dispatch updates otherwise we have a race condition.
-                                    // TODO: switch update is async. Won't we still have a potential
-                                    //       race condition?
-                                    sw.clearAllFlowMods();
-                                    addSwitch(sw);
-                                }
-                            }
-                        } else {
-                            // TODO: Is this the right thing to do if we receive
-                            // some other error besides a bad vendor error?
-                            // Presumably that means the switch did actually
-                            // understand the role request message, but there
-                            // was some other error from processing the message.
-                            // OF 1.2 specifies a OFPET_ROLE_REQUEST_FAILED
-                            // error code, but it doesn't look like the Nicira
-                            // role request has that. Should check OVS source
-                            // code to see if it's possible for any other errors
-                            // to be returned.
-                            // If we received an error the switch is not
-                            // in the correct role, so we need to disconnect it.
-                            // We could also resend the request but then we need to
-                            // check if there are other pending request in which
-                            // case we shouldn't resend. If we do resend we need
-                            // to make sure that the switch eventually accepts one
-                            // of our requests or disconnect the switch. This feels
-                            // cumbersome.
-                            log.debug("Closing {} channel because we recieved an " +
-                                    "error other than BAD_VENDOR", sw);
-                            sw.getChannel().close();
-                        }
-                    }
-                    // Once we support OF 1.2, we'd add code to handle it here.
-                    //if (error.getXid() == state.ofRoleRequestXid) {
-                    //}
-                    if (shouldLogError)
-                        logError(sw, error);
-                    break;
-                case STATS_REPLY:
-                    if (state.hsState.ordinal() <
-                            HandshakeState.FEATURES_REPLY.ordinal()) {
-                        String em = "Unexpected STATS_REPLY from " + sw;
-                        throw new SwitchStateException(em);
-                    }
-                    sw.deliverStatisticsReply(m);
-                    if (sw.hasAttribute(IOFSwitch.SWITCH_DESCRIPTION_FUTURE)) {
-                        processSwitchDescReply();
-                    }
-                    break;
-                case PORT_STATUS:
-                    // We want to update our port state info even if we're in
-                    // the slave role, but we only want to update storage if
-                    // we're the master (or equal).
-                    boolean updateStorage = state.hsState.
-                            equals(HandshakeState.READY) &&
-                            (sw.getRole() != Role.SLAVE);
-                    handlePortStatusMessage(sw, (OFPortStatus) m, updateStorage);
-                    shouldHandleMessage = true;
-                    break;
-
-                default:
-                    shouldHandleMessage = true;
-                    break;
-            }
-
-            if (shouldHandleMessage) {
-                sw.getListenerReadLock().lock();
-                try {
-                    if (sw.isConnected()) {
-                        if (!state.hsState.equals(HandshakeState.READY)) {
-                            log.debug("Ignoring message type {} received " +
-                                    "from switch {} before switch is " +
-                                    "fully configured.", m.getType(), sw);
-                        }
-                        // Check if the controller is in the slave role for the
-                        // switch. If it is, then don't dispatch the message to
-                        // the listeners.
-                        // TODO: Should we dispatch messages that we expect to
-                        // receive when we're in the slave role, e.g. port
-                        // status messages? Since we're "hiding" switches from
-                        // the listeners when we're in the slave role, then it
-                        // seems a little weird to dispatch port status messages
-                        // to them. On the other hand there might be special
-                        // modules that care about all of the connected switches
-                        // and would like to receive port status notifications.
-                        else if (sw.getRole() == Role.SLAVE) {
-                            // Don't log message if it's a port status message
-                            // since we expect to receive those from the switch
-                            // and don't want to emit spurious messages.
-                            if (m.getType() != OFType.PORT_STATUS) {
-                                log.debug("Ignoring message type {} received " +
-                                        "from switch {} while in the slave role.",
-                                        m.getType(), sw);
-                            }
-                        } else {
-                            handleMessage(sw, m, null);
-                        }
-                    }
-                } finally {
-                    sw.getListenerReadLock().unlock();
-                }
-            }
-        }
-    }
-
-    // ****************
-    // Message handlers
-    // ****************
-
-    protected void handlePortStatusMessage(IOFSwitch sw,
-                                           OFPortStatus m,
-                                           boolean updateStorage) {
-        short portNumber = m.getDesc().getPortNumber();
-        OFPhysicalPort port = m.getDesc();
-        if (m.getReason() == (byte) OFPortReason.OFPPR_MODIFY.ordinal()) {
-            boolean portDown = ((OFPortConfig.OFPPC_PORT_DOWN.getValue() & port.getConfig()) > 0) ||
-                    ((OFPortState.OFPPS_LINK_DOWN.getValue() & port.getState()) > 0);
-            sw.setPort(port);
-            if (!portDown) {
-                SwitchUpdate update = new SwitchUpdate(sw, port, SwitchUpdateType.PORTADDED);
-                try {
-                    this.updates.put(update);
-                } catch (InterruptedException e) {
-                    log.error("Failure adding update to queue", e);
-                }
+            if (hasControl) {
+                role = Role.MASTER;
             } else {
-                SwitchUpdate update = new SwitchUpdate(sw, port, SwitchUpdateType.PORTREMOVED);
-                try {
-                    this.updates.put(update);
-                } catch (InterruptedException e) {
-                    log.error("Failure adding update to queue", e);
-                }
+                role = Role.EQUAL; // treat the same as Role.SLAVE
             }
-            //if (updateStorage)
-            //updatePortInfo(sw, port);
-            log.debug("Port #{} modified for {}", portNumber, sw);
-        } else if (m.getReason() == (byte) OFPortReason.OFPPR_ADD.ordinal()) {
-            // XXX Workaround to prevent race condition where a link is detected
-            // and attempted to be written to the database before the port is in
-            // the database. We now suppress link discovery on ports until we're
-            // sure they're in the database.
-            linkDiscovery.disableDiscoveryOnPort(sw.getId(), port.getPortNumber());
 
-            sw.setPort(port);
-            SwitchUpdate update = new SwitchUpdate(sw, port, SwitchUpdateType.PORTADDED);
-            try {
-                this.updates.put(update);
-            } catch (InterruptedException e) {
-                log.error("Failure adding update to queue", e);
+            OFChannelHandler swCh = connectedSwitches.get(dpid.value());
+            if (swCh == null) {
+                log.warn("Switch {} not found in connected switches", dpid);
+                return;
             }
-            //if (updateStorage)
-            //updatePortInfo(sw, port);
-            log.debug("Port #{} added for {}", portNumber, sw);
-        } else if (m.getReason() ==
-                (byte) OFPortReason.OFPPR_DELETE.ordinal()) {
-            sw.deletePort(portNumber);
-            SwitchUpdate update = new SwitchUpdate(sw, port, SwitchUpdateType.PORTREMOVED);
-            try {
-                this.updates.put(update);
-            } catch (InterruptedException e) {
-                log.error("Failure adding update to queue", e);
-            }
-            //if (updateStorage)
-            //removePortInfo(sw, portNumber);
-            log.debug("Port #{} deleted for {}", portNumber, sw);
+
+            log.debug("Sending role request {} msg to {}", role, dpid);
+            swCh.sendRoleRequest(role, RoleRecvStatus.MATCHED_SET_ROLE);
         }
-        SwitchUpdate update = new SwitchUpdate(sw, SwitchUpdateType.PORTCHANGED);
+    }
+
+    public synchronized void submitRegistryRequest(long dpid) {
+        OFChannelHandler h = connectedSwitches.get(dpid);
+        if (h == null) {
+            log.error("Trying to request registry control for switch {} "
+                    + "not in connected switches. Aborting.. ",
+                    HexString.toHexString(dpid));
+            // FIXME shouldn't we immediately return here?
+        }
+        //Request control of the switch from the global registry
         try {
-            this.updates.put(update);
-        } catch (InterruptedException e) {
-            log.error("Failure adding update to queue", e);
+            h.controlRequested = Boolean.TRUE;
+            registryService.requestControl(dpid, new RoleChangeCallback());
+        } catch (RegistryException e) {
+            log.debug("Registry error: {}", e.getMessage());
+            h.controlRequested = Boolean.FALSE;
+        }
+        if (!h.controlRequested) { // XXX what is being attempted here?
+            // yield to allow other thread(s) to release control
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException e) {
+                // Ignore interruptions
+            }
+            // safer to bounce the switch to reconnect here than proceeding further
+            // XXX S why? can't we just try again a little later?
+            log.debug("Closing sw:{} because we weren't able to request control " +
+                    "successfully" + dpid);
+            connectedSwitches.get(dpid).disconnectSwitch();
         }
     }
 
-    /**
-     * flcontext_cache - Keep a thread local stack of contexts
-     */
-    protected static final ThreadLocal<Stack<FloodlightContext>> flcontext_cache =
-            new ThreadLocal<Stack<FloodlightContext>>() {
-                @Override
-                protected Stack<FloodlightContext> initialValue() {
-                    return new Stack<FloodlightContext>();
-                }
-            };
-
-    /**
-     * flcontext_alloc - pop a context off the stack, if required create a new one
-     *
-     * @return FloodlightContext
-     */
-    protected static FloodlightContext flcontext_alloc() {
-        FloodlightContext flcontext = null;
-
-        if (flcontext_cache.get().empty()) {
-            flcontext = new FloodlightContext();
-        } else {
-            flcontext = flcontext_cache.get().pop();
+    public synchronized void releaseRegistryControl(long dpidLong) {
+        OFChannelHandler h = connectedSwitches.get(dpidLong);
+        if (h == null) {
+            log.error("Trying to release registry control for switch {} "
+                    + "not in connected switches. Aborting.. ",
+                    HexString.toHexString(dpidLong));
+            return;
         }
-
-        return flcontext;
+        if (h.controlRequested) {
+            registryService.releaseControl(dpidLong);
+        }
     }
 
-    /**
-     * flcontext_free - Free the context to the current thread
-     *
-     * @param flcontext
-     */
-    protected void flcontext_free(FloodlightContext flcontext) {
-        flcontext.getStorage().clear();
-        flcontext_cache.get().push(flcontext);
-    }
+
+    // *******************
+    // OF Message Handling
+    // *******************
 
     /**
-     * Handle replies to certain OFMessages, and pass others off to listeners
      *
-     * @param sw       The switch for the message
-     * @param m        The message
-     * @param bContext The floodlight context. If null then floodlight context would
-     *                 be allocated in this function
+     * Handle and dispatch a message to IOFMessageListeners.
+     *
+     * We only dispatch messages to listeners if the controller's role is MASTER.
+     *
+     * @param sw The switch sending the message
+     * @param m The message the switch sent
+     * @param flContext The floodlight context to use for this message. If
+     * null, a new context will be allocated.
      * @throws IOException
+     *
+     * FIXME: this method and the ChannelHandler disagree on which messages
+     * should be dispatched and which shouldn't
      */
     @LogMessageDocs({
             @LogMessageDoc(level = "ERROR",
@@ -1210,41 +621,73 @@
                     explanation = "The switch sent a message not handled by " +
                             "the controller")
     })
-    @SuppressWarnings("fallthrough")
+    @SuppressWarnings({ "fallthrough", "unchecked" })
     protected void handleMessage(IOFSwitch sw, OFMessage m,
                                  FloodlightContext bContext)
             throws IOException {
         Ethernet eth = null;
+        short inport = -1;
 
         switch (m.getType()) {
             case PACKET_IN:
                 OFPacketIn pi = (OFPacketIn) m;
-
-                if (pi.getPacketData().length <= 0) {
+                //log.info("saw packet in from sw {}", sw.getStringId());
+                if (pi.getData().length <= 0) {
                     log.error("Ignoring PacketIn (Xid = " + pi.getXid() +
-                            ") because the data field is empty.");
+                            ") because/*  the data field is empty.");
                     return;
                 }
 
+                // get incoming port to store in floodlight context
+                if (sw.getOFVersion() == OFVersion.OF_10) {
+                    inport = pi.getInPort().getShortPortNumber();
+                } else if (sw.getOFVersion() == OFVersion.OF_13) {
+                    for (MatchField<?> mf : pi.getMatch().getMatchFields()) {
+                        if (mf.id == MatchFields.IN_PORT) {
+                            inport = pi.getMatch().get((MatchField<OFPort>)mf)
+                                        .getShortPortNumber();
+                            break;
+                        }
+                    }
+                    if (inport == -1) {
+                        log.error("Match field for incoming port missing in "
+                                + "packet-in from sw {} .. Ignoring msg",
+                                sw.getStringId());
+                        return;
+                    }
+                } else {
+                    // should have been taken care of earlier in handshake
+                    log.error("OFVersion {} not supported for "
+                            + "packet-in from sw {} .. Ignoring msg",
+                            sw.getOFVersion(), sw.getStringId());
+                    return;
+                }
+
+                // decode enclosed ethernet packet to store in floodlight context
                 if (Controller.ALWAYS_DECODE_ETH) {
                     eth = new Ethernet();
-                    eth.deserialize(pi.getPacketData(), 0,
-                            pi.getPacketData().length);
+                    eth.deserialize(pi.getData(), 0,
+                            pi.getData().length);
                 }
                 // fall through to default case...
 
+                /*log.debug("Sw:{} packet-in: {}", sw.getStringId(),
+				String.format("0x%x", eth.getEtherType()));*/
+                if (eth.getEtherType() != (short)EthType.LLDP.getValue())
+			log.trace("Sw:{} packet-in: {}", sw.getStringId(), pi);
+
             default:
 
                 List<IOFMessageListener> listeners = null;
+
                 if (messageListeners.containsKey(m.getType())) {
                     listeners = messageListeners.get(m.getType()).
                             getOrderedListeners();
                 }
-
                 FloodlightContext bc = null;
                 if (listeners != null) {
-                    // Check if floodlight context is passed from the calling 
-                    // function, if so use that floodlight context, otherwise 
+                    // Check if floodlight context is passed from the calling
+                    // function, if so use that floodlight context, otherwise
                     // allocate one
                     if (bContext == null) {
                         bc = flcontext_alloc();
@@ -1256,12 +699,17 @@
                                 IFloodlightProviderService.CONTEXT_PI_PAYLOAD,
                                 eth);
                     }
+                    if (inport != -1) {
+                        bc.getStorage().put(
+                                IFloodlightProviderService.CONTEXT_PI_INPORT,
+                                inport);
+                    }
 
-                    // Get the starting time (overall and per-component) of 
+                    // Get the starting time (overall and per-component) of
                     // the processing chain for this packet if performance
                     // monitoring is turned on
 
-                    Command cmd;
+                    Command cmd = null;
                     for (IOFMessageListener listener : listeners) {
                         if (listener instanceof IOFSwitchFilter) {
                             if (!((IOFSwitchFilter) listener).isInterested(sw)) {
@@ -1269,10 +717,8 @@
                             }
                         }
 
-
                         cmd = listener.receive(sw, m, bc);
 
-
                         if (Command.STOP.equals(cmd)) {
                             break;
                         }
@@ -1286,194 +732,62 @@
         }
     }
 
-    /**
-     * Log an OpenFlow error message from a switch
-     *
-     * @param sw    The switch that sent the error
-     * @param error The error message
-     */
-    @LogMessageDoc(level = "ERROR",
-            message = "Error {error type} {error code} from {switch}",
-            explanation = "The switch responded with an unexpected error" +
-                    "to an OpenFlow message from the controller",
-            recommendation = "This could indicate improper network operation. " +
-                    "If the problem persists restarting the switch and " +
-                    "controller may help."
-    )
-    protected void logError(IOFSwitch sw, OFError error) {
-        int etint = 0xffff & error.getErrorType();
-        if (etint < 0 || etint >= OFErrorType.values().length) {
-            log.error("Unknown error code {} from sw {}", etint, sw);
-        }
-        OFErrorType et = OFErrorType.values()[etint];
-        switch (et) {
-            case OFPET_HELLO_FAILED:
-                OFHelloFailedCode hfc =
-                        OFHelloFailedCode.values()[0xffff & error.getErrorCode()];
-                log.error("Error {} {} from {}", new Object[]{et, hfc, sw});
-                break;
-            case OFPET_BAD_REQUEST:
-                OFBadRequestCode brc =
-                        OFBadRequestCode.values()[0xffff & error.getErrorCode()];
-                log.error("Error {} {} from {}", new Object[]{et, brc, sw});
-                break;
-            case OFPET_BAD_ACTION:
-                OFBadActionCode bac =
-                        OFBadActionCode.values()[0xffff & error.getErrorCode()];
-                log.error("Error {} {} from {}", new Object[]{et, bac, sw});
-                break;
-            case OFPET_FLOW_MOD_FAILED:
-                OFFlowModFailedCode fmfc =
-                        OFFlowModFailedCode.values()[0xffff & error.getErrorCode()];
-                log.error("Error {} {} from {}", new Object[]{et, fmfc, sw});
-                break;
-            case OFPET_PORT_MOD_FAILED:
-                OFPortModFailedCode pmfc =
-                        OFPortModFailedCode.values()[0xffff & error.getErrorCode()];
-                log.error("Error {} {} from {}", new Object[]{et, pmfc, sw});
-                break;
-            case OFPET_QUEUE_OP_FAILED:
-                OFQueueOpFailedCode qofc =
-                        OFQueueOpFailedCode.values()[0xffff & error.getErrorCode()];
-                log.error("Error {} {} from {}", new Object[]{et, qofc, sw});
-                break;
-            default:
-                break;
-        }
-    }
-
-    /**
-     * Add a switch to the active switch list and call the switch listeners.
-     * This happens either when a switch first connects (and the controller is
-     * not in the slave role) or when the role of the controller changes from
-     * slave to master.
-     *
-     * @param sw the switch that has been added
-     */
-    // TODO: need to rethink locking and the synchronous switch update.
-    //       We can / should also handle duplicate DPIDs in connectedSwitches
-    @LogMessageDoc(level = "ERROR",
-            message = "New switch added {switch} for already-added switch {switch}",
-            explanation = "A switch with the same DPID as another switch " +
-                    "connected to the controller.  This can be caused by " +
-                    "multiple switches configured with the same DPID, or " +
-                    "by a switch reconnected very quickly after " +
-                    "disconnecting.",
-            recommendation = "If this happens repeatedly, it is likely there " +
-                    "are switches with duplicate DPIDs on the network.  " +
-                    "Reconfigure the appropriate switches.  If it happens " +
-                    "very rarely, then it is likely this is a transient " +
-                    "network problem that can be ignored."
-    )
-    protected void addSwitch(IOFSwitch sw) {
-        // XXX Workaround to prevent race condition where a link is detected
-        // and attempted to be written to the database before the port is in
-        // the database. We now suppress link discovery on ports until we're
-        // sure they're in the database.
-        for (OFPhysicalPort port : sw.getPorts()) {
-            linkDiscovery.disableDiscoveryOnPort(sw.getId(), port.getPortNumber());
-        }
-
-        // TODO: is it safe to modify the HashMap without holding 
-        // the old switch's lock?
-        OFSwitchImpl oldSw = (OFSwitchImpl) this.activeSwitches.put(sw.getId(), sw);
-        if (sw == oldSw) {
-            // Note == for object equality, not .equals for value
-            log.info("New add switch for pre-existing switch {}", sw);
-            return;
-        }
-
-        if (oldSw != null) {
-            oldSw.getListenerWriteLock().lock();
-            try {
-                log.error("New switch added {} for already-added switch {}",
-                        sw, oldSw);
-                // Set the connected flag to false to suppress calling
-                // the listeners for this switch in processOFMessage
-                oldSw.setConnected(false);
-
-                oldSw.cancelAllStatisticsReplies();
-
-                //updateInactiveSwitchInfo(oldSw);
-
-                // we need to clean out old switch state definitively 
-                // before adding the new switch
-                // FIXME: It seems not completely kosher to call the
-                // switch listeners here. I thought one of the points of
-                // having the asynchronous switch update mechanism was so
-                // the addedSwitch and removedSwitch were always called
-                // from a single thread to simplify concurrency issues
-                // for the listener.
-                if (switchListeners != null) {
-                    for (IOFSwitchListener listener : switchListeners) {
-                        listener.removedSwitch(oldSw);
-                    }
-                }
-                // will eventually trigger a removeSwitch(), which will cause
-                // a "Not removing Switch ... already removed debug message.
-                // TODO: Figure out a way to handle this that avoids the
-                // spurious debug message.
-                log.debug("Closing {} because a new IOFSwitch got added " +
-                        "for this dpid", oldSw);
-                oldSw.getChannel().close();
-            } finally {
-                oldSw.getListenerWriteLock().unlock();
-            }
-        }
-
-        //updateActiveSwitchInfo(sw);
-        SwitchUpdate update = new SwitchUpdate(sw, SwitchUpdateType.ADDED);
-        try {
-            this.updates.put(update);
-        } catch (InterruptedException e) {
-            log.error("Failure adding update to queue", e);
-        }
-    }
-
-    /**
-     * Remove a switch from the active switch list and call the switch listeners.
-     * This happens either when the switch is disconnected or when the
-     * controller's role for the switch changes from master to slave.
-     *
-     * @param sw the switch that has been removed
-     */
-    protected void removeSwitch(IOFSwitch sw) {
-        // No need to acquire the listener lock, since
-        // this method is only called after netty has processed all
-        // pending messages
-        log.debug("removeSwitch: {}", sw);
-        if (!this.activeSwitches.remove(sw.getId(), sw) || !sw.isConnected()) {
-            log.debug("Not removing switch {}; already removed", sw);
-            return;
-        }
-        // We cancel all outstanding statistics replies if the switch transition
-        // from active. In the future we might allow statistics requests 
-        // from slave controllers. Then we need to move this cancelation
-        // to switch disconnect
-        sw.cancelAllStatisticsReplies();
-
-        // FIXME: I think there's a race condition if we call updateInactiveSwitchInfo
-        // here if role support is enabled. In that case if the switch is being
-        // removed because we've been switched to being in the slave role, then I think
-        // it's possible that the new master may have already been promoted to master
-        // and written out the active switch state to storage. If we now execute
-        // updateInactiveSwitchInfo we may wipe out all of the state that was
-        // written out by the new master. Maybe need to revisit how we handle all
-        // of the switch state that's written to storage.
-
-        //updateInactiveSwitchInfo(sw);
-        SwitchUpdate update = new SwitchUpdate(sw, SwitchUpdateType.REMOVED);
-        try {
-            this.updates.put(update);
-        } catch (InterruptedException e) {
-            log.error("Failure adding update to queue", e);
-        }
-    }
-
     // ***************
-    // IFloodlightProvider
+    // IFloodlightProviderService
     // ***************
 
+    // FIXME: remove this method
+    @Override
+    public Map<Long,IOFSwitch> getSwitches() {
+        return getMasterSwitches();
+    }
+
+    // FIXME: remove this method
+    public Map<Long, IOFSwitch> getMasterSwitches() {
+        return Collections.unmodifiableMap(activeMasterSwitches);
+    }
+
+
+    @Override
+    public Set<Long> getAllSwitchDpids() {
+        Set<Long> dpids = new HashSet<Long>();
+        dpids.addAll(activeMasterSwitches.keySet());
+        dpids.addAll(activeEqualSwitches.keySet());
+        return dpids;
+    }
+
+    @Override
+    public Set<Long> getAllMasterSwitchDpids() {
+        Set<Long> dpids = new HashSet<Long>();
+        dpids.addAll(activeMasterSwitches.keySet());
+        return dpids;
+    }
+
+    @Override
+    public Set<Long> getAllEqualSwitchDpids() {
+        Set<Long> dpids = new HashSet<Long>();
+        dpids.addAll(activeEqualSwitches.keySet());
+        return dpids;
+    }
+
+    @Override
+    public IOFSwitch getSwitch(long dpid) {
+        IOFSwitch sw = null;
+        if ((sw = activeMasterSwitches.get(dpid)) != null) return sw;
+        if ((sw = activeEqualSwitches.get(dpid)) != null) return sw;
+        return sw;
+    }
+
+    @Override
+    public IOFSwitch getMasterSwitch(long dpid) {
+        return  activeMasterSwitches.get(dpid);
+    }
+
+    @Override
+    public IOFSwitch getEqualSwitch(long dpid) {
+        return  activeEqualSwitches.get(dpid);
+    }
+
     @Override
     public synchronized void addOFMessageListener(OFType type,
                                                   IOFMessageListener listener) {
@@ -1496,6 +810,10 @@
         }
     }
 
+    public void removeOFMessageListeners(OFType type) {
+        messageListeners.remove(type);
+    }
+
     private void logListeners() {
         for (Map.Entry<OFType,
                 ListenerDispatcher<OFType,
@@ -1507,7 +825,7 @@
                     entry.getValue();
 
             StringBuffer sb = new StringBuffer();
-            sb.append("OFListeners for ");
+            sb.append("OFMessageListeners for ");
             sb.append(type);
             sb.append(": ");
             for (IOFMessageListener l : ldd.getOrderedListeners()) {
@@ -1516,15 +834,14 @@
             }
             log.debug(sb.toString());
         }
-    }
+        StringBuffer sl = new StringBuffer();
+        sl.append("SwitchUpdate Listeners: ");
+        for (IOFSwitchListener swlistener : switchListeners) {
+            sl.append(swlistener.getName());
+            sl.append(",");
+        }
+        log.debug(sl.toString());
 
-    public void removeOFMessageListeners(OFType type) {
-        messageListeners.remove(type);
-    }
-
-    @Override
-    public Map<Long, IOFSwitch> getSwitches() {
-        return Collections.unmodifiableMap(this.activeSwitches);
     }
 
     @Override
@@ -1548,19 +865,7 @@
         return Collections.unmodifiableMap(lers);
     }
 
-    @Override
-    public void addLocalSwitchMastershipListener(
-                ILocalSwitchMastershipListener listener) {
-        this.localSwitchMastershipListeners.addIfAbsent(listener);
-    }
-
-    @Override
-    public void removeLocalSwitchMastershipListener(
-                ILocalSwitchMastershipListener listener) {
-        this.localSwitchMastershipListeners.remove(listener);
-    }
-
-    @Override
+    /*@Override
     @LogMessageDocs({
             @LogMessageDoc(message = "Failed to inject OFMessage {message} onto " +
                     "a null switch",
@@ -1587,7 +892,7 @@
         // discussions it sounds like the right thing to do here would be to
         // inject the message as a netty upstream channel event so it goes
         // through the normal netty event processing, including being
-        // handled 
+        // handled
         if (!activeSwitches.containsKey(sw.getId())) return false;
 
         try {
@@ -1599,53 +904,79 @@
             return false;
         }
         return true;
+    }*/
+
+
+
+//    @Override
+//    public boolean injectOfMessage(IOFSwitch sw, OFMessage msg) {
+//        // call the overloaded version with floodlight context set to null
+//        return injectOfMessage(sw, msg, null);
+//    }
+
+//    @Override
+//    public void handleOutgoingMessage(IOFSwitch sw, OFMessage m,
+//                                      FloodlightContext bc) {
+//
+//        List<IOFMessageListener> listeners = null;
+//        if (messageListeners.containsKey(m.getType())) {
+//            listeners =
+//                    messageListeners.get(m.getType()).getOrderedListeners();
+//        }
+//
+//        if (listeners != null) {
+//            for (IOFMessageListener listener : listeners) {
+//                if (listener instanceof IOFSwitchFilter) {
+//                    if (!((IOFSwitchFilter) listener).isInterested(sw)) {
+//                        continue;
+//                    }
+//                }
+//                if (Command.STOP.equals(listener.receive(sw, m, bc))) {
+//                    break;
+//                }
+//            }
+//        }
+//    }
+
+    @Override
+    public OFFactory getOFMessageFactory_10() {
+        return factory10;
     }
 
     @Override
-    @LogMessageDoc(message = "Calling System.exit",
-            explanation = "The controller is terminating")
-    public synchronized void terminate() {
-        log.info("Calling System.exit");
-        System.exit(1);
+    public OFFactory getOFMessageFactory_13() {
+        return factory13;
     }
 
     @Override
-    public boolean injectOfMessage(IOFSwitch sw, OFMessage msg) {
-        // call the overloaded version with floodlight context set to null    
-        return injectOfMessage(sw, msg, null);
-    }
-
-    @Override
-    public void handleOutgoingMessage(IOFSwitch sw, OFMessage m,
-                                      FloodlightContext bc) {
-        if (log.isTraceEnabled()) {
-            String str = OFMessage.getDataAsString(sw, m, bc);
-            log.trace("{}", str);
-        }
-
-        List<IOFMessageListener> listeners = null;
-        if (messageListeners.containsKey(m.getType())) {
-            listeners =
-                    messageListeners.get(m.getType()).getOrderedListeners();
-        }
-
-        if (listeners != null) {
-            for (IOFMessageListener listener : listeners) {
-                if (listener instanceof IOFSwitchFilter) {
-                    if (!((IOFSwitchFilter) listener).isInterested(sw)) {
-                        continue;
-                    }
-                }
-                if (Command.STOP.equals(listener.receive(sw, m, bc))) {
-                    break;
-                }
-            }
+    public void publishUpdate(IUpdate update) {
+        try {
+            this.updates.put(update);
+        } catch (InterruptedException e) {
+            log.error("Failure adding update to queue", e);
         }
     }
 
     @Override
-    public BasicFactory getOFMessageFactory() {
-        return factory;
+    public Map<String, String> getControllerNodeIPs() {
+        // We return a copy of the mapping so we can guarantee that
+        // the mapping return is the same as one that will be (or was)
+        // dispatched to IHAListeners
+        HashMap<String, String> retval = new HashMap<String, String>();
+        synchronized (controllerNodeIPsCache) {
+            retval.putAll(controllerNodeIPsCache);
+        }
+        return retval;
+    }
+
+    @Override
+    public long getSystemStartTime() {
+        return (this.systemStartTime);
+    }
+
+    @Override
+    public void setAlwaysClearFlowsOnSwAdd(boolean value) {
+        this.alwaysClearFlowsOnSwAdd = value;
     }
 
     @Override
@@ -1653,10 +984,26 @@
         return onosInstanceId;
     }
 
+    /**
+     * FOR TESTING ONLY. Dispatch all updates in the update queue until queue is
+     * empty
+     */
+    void processUpdateQueueForTesting() {
+        while (!updates.isEmpty()) {
+            IUpdate update = updates.poll();
+            if (update != null)
+                update.dispatch();
+        }
+    }
+
+
     // **************
     // Initialization
     // **************
 
+    // XXX S This should probably go away OR it should be edited to handle
+    // controller roles per switch! Then it could be a way to
+    // deterministically configure a switch to a MASTER controller instance
     /**
      * Sets the initial role based on properties in the config params.
      * It looks for two different properties.
@@ -1737,6 +1084,7 @@
      *
      * @throws IOException
      */
+    @Override
     @LogMessageDocs({
             @LogMessageDoc(message = "Listening for switch connections on {address}",
                     explanation = "The controller is ready and listening for new" +
@@ -1781,7 +1129,9 @@
                 IUpdate update = updates.take();
                 update.dispatch();
             } catch (InterruptedException e) {
-                return;
+                log.error("Received interrupted exception in updates loop;" +
+                          "terminating process");
+                terminate();
             } catch (Exception e) {
                 log.error("Exception in controller updates loop", e);
             }
@@ -1829,23 +1179,6 @@
         log.debug("ControllerId set to {}", this.onosInstanceId);
     }
 
-    private void initVendorMessages() {
-        // Configure openflowj to be able to parse the role request/reply
-        // vendor messages.
-        OFBasicVendorId niciraVendorId = new OFBasicVendorId(
-                OFNiciraVendorData.NX_VENDOR_ID, 4);
-        OFVendorId.registerVendorId(niciraVendorId);
-        OFBasicVendorDataType roleRequestVendorData =
-                new OFBasicVendorDataType(
-                        OFRoleRequestVendorData.NXT_ROLE_REQUEST,
-                        OFRoleRequestVendorData.getInstantiable());
-        niciraVendorId.registerVendorDataType(roleRequestVendorData);
-        OFBasicVendorDataType roleReplyVendorData =
-                new OFBasicVendorDataType(
-                        OFRoleReplyVendorData.NXT_ROLE_REPLY,
-                        OFRoleReplyVendorData.getInstantiable());
-        niciraVendorId.registerVendorDataType(roleReplyVendorData);
-    }
 
     /**
      * Initialize internal data structures
@@ -1854,32 +1187,40 @@
         // These data structures are initialized here because other
         // module's startUp() might be called before ours
         this.messageListeners =
-                new ConcurrentHashMap<OFType,
-                        ListenerDispatcher<OFType,
-                                IOFMessageListener>>();
+                new ConcurrentHashMap<OFType, ListenerDispatcher<OFType,
+                                                    IOFMessageListener>>();
         this.switchListeners = new CopyOnWriteArraySet<IOFSwitchListener>();
-        this.activeSwitches = new ConcurrentHashMap<Long, IOFSwitch>();
-        this.connectedSwitches = new HashSet<OFSwitchImpl>();
+        this.activeMasterSwitches = new ConcurrentHashMap<Long, IOFSwitch>();
+        this.activeEqualSwitches = new ConcurrentHashMap<Long, IOFSwitch>();
+        this.connectedSwitches = new ConcurrentHashMap<Long, OFChannelHandler>();
         this.controllerNodeIPsCache = new HashMap<String, String>();
         this.updates = new LinkedBlockingQueue<IUpdate>();
-        this.factory = new BasicFactory();
+
         setConfigParams(configParams);
-        //Set the controller's role to MASTER so it always tries to do role requests.
-        this.role = Role.MASTER;
-        this.roleChanger = new RoleChanger();
-        initVendorMessages();
         this.systemStartTime = System.currentTimeMillis();
+        this.counters = new Counters();
+        this.multiCacheLock = new Object();
+
+        String option = configParams.get("flushSwitchesOnReconnect");
+        if (option != null && option.equalsIgnoreCase("true")) {
+            this.setAlwaysClearFlowsOnSwActivate(true);
+            log.info("Flush switches on reconnect -- Enabled.");
+        } else {
+            this.setAlwaysClearFlowsOnSwActivate(false);
+            log.info("Flush switches on reconnect -- Disabled");
+        }
     }
 
     /**
      * Startup all of the controller's components
+     * @throws FloodlightModuleException
      */
     @LogMessageDoc(message = "Waiting for storage source",
             explanation = "The system database is not yet ready",
             recommendation = "If this message persists, this indicates " +
                     "that the system database has failed to start. " +
                     LogMessageDoc.CHECK_CONTROLLER)
-    public void startupComponents() {
+    public void startupComponents() throws FloodlightModuleException {
         try {
             registryService.registerController(onosInstanceId.toString());
         } catch (RegistryException e) {
@@ -1888,31 +1229,626 @@
 
         // Add our REST API
         restApi.addRestletRoutable(new CoreWebRoutable());
-    }
 
-    @Override
-    public Map<String, String> getControllerNodeIPs() {
-        // We return a copy of the mapping so we can guarantee that
-        // the mapping return is the same as one that will be (or was)
-        // dispatched to IHAListeners
-        HashMap<String, String> retval = new HashMap<String, String>();
-        synchronized (controllerNodeIPsCache) {
-            retval.putAll(controllerNodeIPsCache);
+        // Startup load monitoring
+        if (overload_drop) {
+            this.loadmonitor.startMonitoring(
+                this.threadPool.getScheduledExecutor());
         }
-        return retval;
+
+        // register counters and events
+        try {
+            this.counters.createCounters(debugCounters);
+        } catch (CounterException e) {
+            throw new FloodlightModuleException(e.getMessage());
+        }
+        registerControllerDebugEvents();
+    }
+
+    // **************
+    // debugCounter registrations
+    // **************
+
+    public static class Counters {
+        public static final String prefix = "controller";
+        public IDebugCounter setRoleEqual;
+        public IDebugCounter setSameRole;
+        public IDebugCounter setRoleMaster;
+        public IDebugCounter remoteStoreNotification;
+        public IDebugCounter invalidPortsChanged;
+        public IDebugCounter invalidSwitchActivatedWhileSlave;
+        public IDebugCounter invalidStoreEventWhileMaster;
+        public IDebugCounter switchDisconnectedWhileSlave;
+        public IDebugCounter switchActivated;
+        public IDebugCounter errorSameSwitchReactivated; // err
+        public IDebugCounter switchWithSameDpidActivated; // warn
+        public IDebugCounter newSwitchActivated;   // new switch
+        public IDebugCounter syncedSwitchActivated;
+        public IDebugCounter readyForReconcile;
+        public IDebugCounter newSwitchFromStore;
+        public IDebugCounter updatedSwitchFromStore;
+        public IDebugCounter switchDisconnected;
+        public IDebugCounter syncedSwitchRemoved;
+        public IDebugCounter unknownSwitchRemovedFromStore;
+        public IDebugCounter consolidateStoreRunCount;
+        public IDebugCounter consolidateStoreInconsistencies;
+        public IDebugCounter storeSyncError;
+        public IDebugCounter switchesNotReconnectingToNewMaster;
+        public IDebugCounter switchPortChanged;
+        public IDebugCounter switchOtherChange;
+        public IDebugCounter dispatchMessageWhileSlave;
+        public IDebugCounter dispatchMessage;  // does this cnt make sense? more specific?? per type? count stops?
+        public IDebugCounter controllerNodeIpsChanged;
+        public IDebugCounter messageReceived;
+        public IDebugCounter messageInputThrottled;
+        public IDebugCounter switchDisconnectReadTimeout;
+        public IDebugCounter switchDisconnectHandshakeTimeout;
+        public IDebugCounter switchDisconnectIOError;
+        public IDebugCounter switchDisconnectParseError;
+        public IDebugCounter switchDisconnectSwitchStateException;
+        public IDebugCounter rejectedExecutionException;
+        public IDebugCounter switchDisconnectOtherException;
+        public IDebugCounter switchConnected;
+        public IDebugCounter unhandledMessage;
+        public IDebugCounter packetInWhileSwitchIsSlave;
+        public IDebugCounter epermErrorWhileSwitchIsMaster;
+        public IDebugCounter roleNotResentBecauseRolePending;
+        public IDebugCounter roleRequestSent;
+        public IDebugCounter roleReplyTimeout;
+        public IDebugCounter roleReplyReceived; // expected RoleReply received
+        public IDebugCounter roleReplyErrorUnsupported;
+        public IDebugCounter switchCounterRegistrationFailed;
+        public IDebugCounter packetParsingError;
+
+        void createCounters(IDebugCounterService debugCounters) throws CounterException {
+            setRoleEqual =
+                debugCounters.registerCounter(
+                            prefix, "set-role-equal",
+                            "Controller received a role request with role of "+
+                            "EQUAL which is unusual",
+                            CounterType.ALWAYS_COUNT);
+            setSameRole =
+                debugCounters.registerCounter(
+                            prefix, "set-same-role",
+                            "Controller received a role request for the same " +
+                            "role the controller already had",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            setRoleMaster =
+                debugCounters.registerCounter(
+                            prefix, "set-role-master",
+                            "Controller received a role request with role of " +
+                            "MASTER. This counter can be at most 1.",
+                            CounterType.ALWAYS_COUNT);
+
+            remoteStoreNotification =
+                debugCounters.registerCounter(
+                            prefix, "remote-store-notification",
+                            "Received a notification from the sync service " +
+                            "indicating that switch information has changed",
+                            CounterType.ALWAYS_COUNT);
+
+            invalidPortsChanged =
+                debugCounters.registerCounter(
+                            prefix, "invalid-ports-changed",
+                            "Received an unexpected ports changed " +
+                            "notification while the controller was in " +
+                            "SLAVE role.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            invalidSwitchActivatedWhileSlave =
+                debugCounters.registerCounter(
+                            prefix, "invalid-switch-activated-while-slave",
+                            "Received an unexpected switchActivated " +
+                            "notification while the controller was in " +
+                            "SLAVE role.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            invalidStoreEventWhileMaster =
+                debugCounters.registerCounter(
+                            prefix, "invalid-store-event-while-master",
+                            "Received an unexpected notification from " +
+                            "the sync store while the controller was in " +
+                            "MASTER role.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            switchDisconnectedWhileSlave =
+                debugCounters.registerCounter(
+                            prefix, "switch-disconnected-while-slave",
+                            "A switch disconnected and the controller was " +
+                            "in SLAVE role.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            switchActivated =
+                debugCounters.registerCounter(
+                            prefix, "switch-activated",
+                            "A switch connected to this controller is now " +
+                            "in MASTER role",
+                            CounterType.ALWAYS_COUNT);
+
+            errorSameSwitchReactivated = // err
+                debugCounters.registerCounter(
+                            prefix, "error-same-switch-reactivated",
+                            "A switch that was already in active state " +
+                            "was activated again. This indicates a " +
+                            "controller defect",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+
+            switchWithSameDpidActivated = // warn
+                debugCounters.registerCounter(
+                            prefix, "switch-with-same-dpid-activated",
+                            "A switch with the same DPID as another switch " +
+                            "connected to the controller. This can be " +
+                            "caused by multiple switches configured with " +
+                            "the same DPID or by a switch reconnecting very " +
+                            "quickly.",
+                            CounterType.COUNT_ON_DEMAND,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            newSwitchActivated =   // new switch
+                debugCounters.registerCounter(
+                            prefix, "new-switch-activated",
+                            "A new switch has completed the handshake as " +
+                            "MASTER. The switch was not known to any other " +
+                            "controller in the cluster",
+                            CounterType.ALWAYS_COUNT);
+            syncedSwitchActivated =
+                debugCounters.registerCounter(
+                            prefix, "synced-switch-activated",
+                            "A switch has completed the handshake as " +
+                            "MASTER. The switch was known to another " +
+                            "controller in the cluster",
+                            CounterType.ALWAYS_COUNT);
+
+            readyForReconcile =
+                debugCounters.registerCounter(
+                            prefix, "ready-for-reconcile",
+                            "Controller is ready for flow reconciliation " +
+                            "after Slave to Master transition. Either all " +
+                            "previously known switches are now active " +
+                            "or they have timed out and have been removed." +
+                            "This counter will be 0 or 1.",
+                            CounterType.ALWAYS_COUNT);
+
+            newSwitchFromStore =
+                debugCounters.registerCounter(
+                            prefix, "new-switch-from-store",
+                            "A new switch has connected to another " +
+                            "another controller in the cluster. This " +
+                            "controller instance has received a sync store " +
+                            "notification for it.",
+                            CounterType.ALWAYS_COUNT);
+
+            updatedSwitchFromStore =
+                debugCounters.registerCounter(
+                            prefix, "updated-switch-from-store",
+                            "Information about a switch connected to " +
+                            "another controller instance was updated in " +
+                            "the sync store. This controller instance has " +
+                            "received a notification for it",
+                            CounterType.ALWAYS_COUNT);
+
+            switchDisconnected =
+                debugCounters.registerCounter(
+                            prefix, "switch-disconnected",
+                            "FIXME: switch has disconnected",
+                            CounterType.ALWAYS_COUNT);
+
+            syncedSwitchRemoved =
+                debugCounters.registerCounter(
+                            prefix, "synced-switch-removed",
+                            "A switch connected to another controller " +
+                            "instance has disconnected from the controller " +
+                            "cluster. This controller instance has " +
+                            "received a notification for it",
+                            CounterType.ALWAYS_COUNT);
+
+            unknownSwitchRemovedFromStore =
+                debugCounters.registerCounter(
+                            prefix, "unknown-switch-removed-from-store",
+                            "This controller instances has received a sync " +
+                            "store notification that a switch has " +
+                            "disconnected but this controller instance " +
+                            "did not have the any information about the " +
+                            "switch", // might be less than warning
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            consolidateStoreRunCount =
+                debugCounters.registerCounter(
+                            prefix, "consolidate-store-run-count",
+                            "This controller has transitioned from SLAVE " +
+                            "to MASTER and waited for switches to reconnect. " +
+                            "The controller has finished waiting and has " +
+                            "reconciled switch entries in the sync store " +
+                            "with live state",
+                            CounterType.ALWAYS_COUNT);
+
+            consolidateStoreInconsistencies =
+                    debugCounters.registerCounter(
+                                prefix, "consolidate-store-inconsistencies",
+                                "During switch sync store consolidation: " +
+                                "Number of switches that were in the store " +
+                                "but not otherwise known plus number of " +
+                                "switches that were in the store previously " +
+                                "but are now missing plus number of "  +
+                                "connected switches that were absent from " +
+                                "the store although this controller has " +
+                                "written them. A non-zero count " +
+                                "indicates a brief split-brain dual MASTER " +
+                                "situation during fail-over",
+                                CounterType.ALWAYS_COUNT);
+
+            storeSyncError =
+                debugCounters.registerCounter(
+                            prefix, "store-sync-error",
+                            "Number of times a sync store operation failed " +
+                            "due to a store sync exception or an entry in " +
+                            "in the store had invalid data.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+
+            switchesNotReconnectingToNewMaster =
+                debugCounters.registerCounter(
+                            prefix, "switches-not-reconnecting-to-new-master",
+                            "Switches that were connected to another " +
+                            "controller instance in the cluster but that " +
+                            "did not reconnect to this controller after it " +
+                            "transitioned to MASTER", // might be less than warning
+                            CounterType.ALWAYS_COUNT);
+
+            switchPortChanged =
+                debugCounters.registerCounter(
+                            prefix, "switch-port-changed",
+                            "Number of times switch ports have changed",
+                            CounterType.ALWAYS_COUNT);
+            switchOtherChange =
+                debugCounters.registerCounter(
+                            prefix, "switch-other-change",
+                            "Number of times other information of a switch " +
+                            "has changed.",
+                            CounterType.ALWAYS_COUNT);
+
+            dispatchMessageWhileSlave =
+                debugCounters.registerCounter(
+                            prefix, "dispatch-message-while-slave",
+                            "Number of times an OF message was received " +
+                            "and supposed to be dispatched but the " +
+                            "controller was in SLAVE role and the message " +
+                            "was not dispatched",
+                            CounterType.ALWAYS_COUNT);
+
+            dispatchMessage =  // does this cnt make sense? more specific?? per type? count stops?
+                debugCounters.registerCounter(
+                            prefix, "dispatch-message",
+                            "Number of times an OF message was dispatched " +
+                            "to registered modules",
+                            CounterType.ALWAYS_COUNT);
+
+            controllerNodeIpsChanged =
+                debugCounters.registerCounter(
+                            prefix, "controller-nodes-ips-changed",
+                            "IP addresses of controller nodes have changed",
+                            CounterType.ALWAYS_COUNT);
+
+        //------------------------
+        // channel handler counters. Factor them out ??
+            messageReceived =
+                debugCounters.registerCounter(
+                            prefix, "message-received",
+                            "Number of OpenFlow messages received. Some of " +
+                            "these might be throttled",
+                            CounterType.ALWAYS_COUNT);
+            messageInputThrottled =
+                debugCounters.registerCounter(
+                            prefix, "message-input-throttled",
+                            "Number of OpenFlow messages that were " +
+                            "throttled due to high load from the sender",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+        // TODO: more counters in messageReceived ??
+
+            switchDisconnectReadTimeout =
+                debugCounters.registerCounter(
+                            prefix, "switch-disconnect-read-timeout",
+                            "Number of times a switch was disconnected due " +
+                            "due the switch failing to send OpenFlow " +
+                            "messages or responding to OpenFlow ECHOs",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+            switchDisconnectHandshakeTimeout =
+                debugCounters.registerCounter(
+                            prefix, "switch-disconnect-handshake-timeout",
+                            "Number of times a switch was disconnected " +
+                            "because it failed to complete the handshake " +
+                            "in time.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+            switchDisconnectIOError =
+                debugCounters.registerCounter(
+                            prefix, "switch-disconnect-io-error",
+                            "Number of times a switch was disconnected " +
+                            "due to IO errors on the switch connection.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+            switchDisconnectParseError =
+                debugCounters.registerCounter(
+                            prefix, "switch-disconnect-parse-error",
+                           "Number of times a switch was disconnected " +
+                           "because it sent an invalid packet that could " +
+                           "not be parsed",
+                           CounterType.ALWAYS_COUNT,
+                           IDebugCounterService.CTR_MDATA_ERROR);
+
+            switchDisconnectSwitchStateException =
+                debugCounters.registerCounter(
+                            prefix, "switch-disconnect-switch-state-exception",
+                            "Number of times a switch was disconnected " +
+                            "because it sent messages that were invalid " +
+                            "given the switch connection's state.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+            rejectedExecutionException =
+                debugCounters.registerCounter(
+                            prefix, "rejected-execution-exception",
+                            "TODO",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+
+            switchDisconnectOtherException =
+                debugCounters.registerCounter(
+                            prefix,  "switch-disconnect-other-exception",
+                            "Number of times a switch was disconnected " +
+                            "due to an exceptional situation not covered " +
+                            "by other counters",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_ERROR);
+
+            switchConnected =
+                debugCounters.registerCounter(
+                            prefix, "switch-connected",
+                            "Number of times a new switch connection was " +
+                            "established",
+                            CounterType.ALWAYS_COUNT);
+
+            unhandledMessage =
+                debugCounters.registerCounter(
+                            prefix, "unhandled-message",
+                            "Number of times an OpenFlow message was " +
+                            "received that the controller ignored because " +
+                            "it was inapproriate given the switch " +
+                            "connection's state.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+                            // might be less than warning
+
+            packetInWhileSwitchIsSlave =
+                debugCounters.registerCounter(
+                            prefix, "packet-in-while-switch-is-slave",
+                            "Number of times a packet in was received " +
+                            "from a switch that was in SLAVE role. " +
+                            "Possibly inidicates inconsistent roles.",
+                            CounterType.ALWAYS_COUNT);
+            epermErrorWhileSwitchIsMaster =
+                debugCounters.registerCounter(
+                            prefix, "eperm-error-while-switch-is-master",
+                            "Number of times a permission error was " +
+                            "received while the switch was in MASTER role. " +
+                            "Possibly inidicates inconsistent roles.",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            roleNotResentBecauseRolePending =
+                debugCounters.registerCounter(
+                            prefix, "role-not-resent-because-role-pending",
+                            "The controller tried to reestablish a role " +
+                            "with a switch but did not do so because a " +
+                            "previous role request was still pending",
+                            CounterType.ALWAYS_COUNT);
+            roleRequestSent =
+                debugCounters.registerCounter(
+                            prefix, "role-request-sent",
+                            "Number of times the controller sent a role " +
+                            "request to a switch.",
+                            CounterType.ALWAYS_COUNT);
+            roleReplyTimeout =
+                debugCounters.registerCounter(
+                            prefix, "role-reply-timeout",
+                            "Number of times a role request message did not " +
+                            "receive the expected reply from a switch",
+                            CounterType.ALWAYS_COUNT,
+                            IDebugCounterService.CTR_MDATA_WARN);
+
+            roleReplyReceived = // expected RoleReply received
+                debugCounters.registerCounter(
+                            prefix, "role-reply-received",
+                            "Number of times the controller received the " +
+                            "expected role reply message from a switch",
+                            CounterType.ALWAYS_COUNT);
+
+            roleReplyErrorUnsupported =
+                debugCounters.registerCounter(
+                            prefix, "role-reply-error-unsupported",
+                            "Number of times the controller received an " +
+                            "error from a switch in response to a role " +
+                            "request indicating that the switch does not " +
+                            "support roles.",
+                            CounterType.ALWAYS_COUNT);
+
+            switchCounterRegistrationFailed =
+                    debugCounters.registerCounter(prefix,
+                                "switch-counter-registration-failed",
+                                "Number of times the controller failed to " +
+                                "register per-switch debug counters",
+                                CounterType.ALWAYS_COUNT,
+                                IDebugCounterService.CTR_MDATA_WARN);
+
+            packetParsingError =
+                    debugCounters.registerCounter(prefix,
+                                "packet-parsing-error",
+                                "Number of times the packet parsing " +
+                                "encountered an error",
+                                CounterType.ALWAYS_COUNT,
+                                IDebugCounterService.CTR_MDATA_ERROR);
+        }
     }
 
     @Override
-    public long getSystemStartTime() {
-        return (this.systemStartTime);
+    public Counters getCounters() {
+        return this.counters;
+    }
+
+    // **************
+    // debugEvent registrations
+    // **************
+
+    private void registerControllerDebugEvents() throws FloodlightModuleException {
+        if (debugEvents == null) {
+            debugEvents = new NullDebugEvent();
+        }
+        try {
+            evSwitch = debugEvents.registerEvent(
+                               Counters.prefix, "switchevent",
+                               "Switch connected, disconnected or port changed",
+                               EventType.ALWAYS_LOG, SwitchEvent.class, 100);
+        } catch (MaxEventsRegistered e) {
+            throw new FloodlightModuleException("Max events registered", e);
+        }
+    }
+
+    public class SwitchEvent {
+        @EventColumn(name = "dpid", description = EventFieldType.DPID)
+        long dpid;
+
+        @EventColumn(name = "reason", description = EventFieldType.STRING)
+        String reason;
+
+        public SwitchEvent(long dpid, String reason) {
+            this.dpid = dpid;
+            this.reason = reason;
+        }
+    }
+
+    // **************
+    // Utility methods
+    // **************
+
+    @Override
+    public void setAlwaysClearFlowsOnSwActivate(boolean value) {
+        //this.alwaysClearFlowsOnSwActivate = value;
+        // XXX S need to be a little more careful about this
     }
 
     @Override
-    public void setAlwaysClearFlowsOnSwAdd(boolean value) {
-        this.alwaysClearFlowsOnSwAdd = value;
+    public Map<String, Long> getMemory() {
+        Map<String, Long> m = new HashMap<String, Long>();
+        Runtime runtime = Runtime.getRuntime();
+        m.put("total", runtime.totalMemory());
+        m.put("free", runtime.freeMemory());
+        return m;
     }
 
-    public boolean getAlwaysClearFlowsOnSwAdd() {
-        return this.alwaysClearFlowsOnSwAdd;
+    @Override
+    public Long getUptime() {
+        RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
+        return rb.getUptime();
     }
+
+    /**
+     * Forward to the driver-manager to get an IOFSwitch instance.
+     * @param desc
+     * @return
+     */
+    protected IOFSwitch getOFSwitchInstance(OFDescStatsReply desc, OFVersion ofv) {
+        return DriverManager.getOFSwitchImpl(desc, ofv);
+    }
+
+    protected IThreadPoolService getThreadPoolService() {
+        return this.threadPool;
+    }
+
+    /**
+     * Part of the controller updates framework (see 'run()' method)
+     * Use this method to add an IUpdate. A thread-pool will serve the update
+     * by dispatching it to all listeners for that update.
+     * @param update
+     */
+    @LogMessageDoc(level="WARN",
+            message="Failure adding update {} to queue",
+            explanation="The controller tried to add an internal notification" +
+                        " to its message queue but the add failed.",
+            recommendation=LogMessageDoc.REPORT_CONTROLLER_BUG)
+    private void addUpdateToQueue(IUpdate update) {
+        try {
+            this.updates.put(update);
+        } catch (InterruptedException e) {
+            // This should never happen
+            log.error("Failure adding update {} to queue.", update);
+        }
+    }
+
+    void flushAll() {
+        // Flush all flow-mods/packet-out/stats generated from this "train"
+        OFSwitchImplBase.flush_all();
+        debugCounters.flushCounters();
+        debugEvents.flushEvents();
+    }
+
+    /**
+     * flcontext_free - Free the context to the current thread
+     *
+     * @param flcontext
+     */
+    protected void flcontext_free(FloodlightContext flcontext) {
+        flcontext.getStorage().clear();
+        flcontext_cache.get().push(flcontext);
+    }
+
+    @LogMessageDoc(message = "Calling System.exit",
+            explanation = "The controller is terminating")
+    private synchronized void terminate() {
+        log.info("Calling System.exit");
+        System.exit(1);
+    }
+
+
+    // ***************
+    // Floodlight context related
+    // ***************
+
+    /**
+     * flcontext_cache - Keep a thread local stack of contexts
+     */
+    protected static final ThreadLocal<Stack<FloodlightContext>> flcontext_cache =
+            new ThreadLocal<Stack<FloodlightContext>>() {
+                @Override
+                protected Stack<FloodlightContext> initialValue() {
+                    return new Stack<FloodlightContext>();
+                }
+            };
+
+    /**
+     * flcontext_alloc - pop a context off the stack, if required create a new one
+     *
+     * @return FloodlightContext
+     */
+    protected static FloodlightContext flcontext_alloc() {
+        FloodlightContext flcontext = null;
+
+        if (flcontext_cache.get().empty()) {
+            flcontext = new FloodlightContext();
+        } else {
+            flcontext = flcontext_cache.get().pop();
+        }
+
+        return flcontext;
+    }
+
+
 }
diff --git a/src/main/java/net/floodlightcontroller/core/internal/HandshakeTimeoutHandler.java b/src/main/java/net/floodlightcontroller/core/internal/HandshakeTimeoutHandler.java
index d5950b4..c9aa3ca 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/HandshakeTimeoutHandler.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/HandshakeTimeoutHandler.java
@@ -1,31 +1,28 @@
 /**
- *    Copyright 2011, Big Switch Networks, Inc.
- *    Originally created by David Erickson, Stanford University
- *
- *    Licensed under the Apache License, Version 2.0 (the "License"); you may
- *    not use this file except in compliance with the License. You may obtain
- *    a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- *    License for the specific language governing permissions and limitations
- *    under the License.
- **/
+*    Copyright 2011, Big Switch Networks, Inc.
+*    Originally created by David Erickson, Stanford University
+*
+*    Licensed under the Apache License, Version 2.0 (the "License"); you may
+*    not use this file except in compliance with the License. You may obtain
+*    a copy of the License at
+*
+*         http://www.apache.org/licenses/LICENSE-2.0
+*
+*    Unless required by applicable law or agreed to in writing, software
+*    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+*    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+*    License for the specific language governing permissions and limitations
+*    under the License.
+**/
 
 package net.floodlightcontroller.core.internal;
 
 import java.util.concurrent.TimeUnit;
 
-import net.floodlightcontroller.core.internal.OFChannelState.HandshakeState;
-
 import org.jboss.netty.channel.ChannelHandlerContext;
 import org.jboss.netty.channel.ChannelStateEvent;
 import org.jboss.netty.channel.Channels;
 import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
-import org.jboss.netty.util.ExternalResourceReleasable;
 import org.jboss.netty.util.Timeout;
 import org.jboss.netty.util.Timer;
 import org.jboss.netty.util.TimerTask;
@@ -34,20 +31,20 @@
  * Trigger a timeout if a switch fails to complete handshake soon enough
  */
 public class HandshakeTimeoutHandler
-        extends SimpleChannelUpstreamHandler
-        implements ExternalResourceReleasable {
+    extends SimpleChannelUpstreamHandler {
     static final HandshakeTimeoutException EXCEPTION =
             new HandshakeTimeoutException();
 
-    final OFChannelState state;
+    final OFChannelHandler channelHandler;
     final Timer timer;
     final long timeoutNanos;
     volatile Timeout timeout;
 
-    public HandshakeTimeoutHandler(OFChannelState state, Timer timer,
+    public HandshakeTimeoutHandler(OFChannelHandler channelHandler,
+                                   Timer timer,
                                    long timeoutSeconds) {
         super();
-        this.state = state;
+        this.channelHandler = channelHandler;
         this.timer = timer;
         this.timeoutNanos = TimeUnit.SECONDS.toNanos(timeoutSeconds);
 
@@ -58,7 +55,7 @@
             throws Exception {
         if (timeoutNanos > 0) {
             timeout = timer.newTimeout(new HandshakeTimeoutTask(ctx),
-                    timeoutNanos, TimeUnit.NANOSECONDS);
+                                       timeoutNanos, TimeUnit.NANOSECONDS);
         }
         ctx.sendUpstream(e);
     }
@@ -72,11 +69,6 @@
         }
     }
 
-    @Override
-    public void releaseExternalResources() {
-        timer.stop();
-    }
-
     private final class HandshakeTimeoutTask implements TimerTask {
 
         private final ChannelHandlerContext ctx;
@@ -94,7 +86,7 @@
             if (!ctx.getChannel().isOpen()) {
                 return;
             }
-            if (!state.hsState.equals(HandshakeState.READY))
+            if (!channelHandler.isHandshakeComplete())
                 Channels.fireExceptionCaught(ctx, EXCEPTION);
         }
     }
diff --git a/src/main/java/net/floodlightcontroller/core/internal/IOFSwitchFeatures.java b/src/main/java/net/floodlightcontroller/core/internal/IOFSwitchFeatures.java
deleted file mode 100644
index f537868..0000000
--- a/src/main/java/net/floodlightcontroller/core/internal/IOFSwitchFeatures.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package net.floodlightcontroller.core.internal;
-
-import net.floodlightcontroller.core.IOFSwitch;
-
-import org.openflow.protocol.statistics.OFDescriptionStatistics;
-
-public interface IOFSwitchFeatures {
-    public void setFromDescription(IOFSwitch sw, OFDescriptionStatistics description);
-}
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFChannelHandler.java b/src/main/java/net/floodlightcontroller/core/internal/OFChannelHandler.java
new file mode 100644
index 0000000..a79e7d9
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/core/internal/OFChannelHandler.java
@@ -0,0 +1,2251 @@
+package net.floodlightcontroller.core.internal;
+
+import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.RejectedExecutionException;
+
+import net.floodlightcontroller.core.FloodlightContext;
+import net.floodlightcontroller.core.IFloodlightProviderService.Role;
+import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.IOFSwitch.PortChangeEvent;
+import net.floodlightcontroller.core.annotations.LogMessageDoc;
+import net.floodlightcontroller.core.annotations.LogMessageDocs;
+import net.floodlightcontroller.core.internal.Controller.Counters;
+import net.floodlightcontroller.core.internal.OFChannelHandler.ChannelState.RoleReplyInfo;
+import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterException;
+import net.floodlightcontroller.util.LoadMonitor;
+
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
+import org.jboss.netty.handler.timeout.IdleStateEvent;
+import org.jboss.netty.handler.timeout.ReadTimeoutException;
+import org.projectfloodlight.openflow.exceptions.OFParseError;
+import org.projectfloodlight.openflow.protocol.OFAsyncGetReply;
+import org.projectfloodlight.openflow.protocol.OFBadRequestCode;
+import org.projectfloodlight.openflow.protocol.OFBarrierReply;
+import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
+import org.projectfloodlight.openflow.protocol.OFControllerRole;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFDescStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFEchoReply;
+import org.projectfloodlight.openflow.protocol.OFEchoRequest;
+import org.projectfloodlight.openflow.protocol.OFErrorMsg;
+import org.projectfloodlight.openflow.protocol.OFErrorType;
+import org.projectfloodlight.openflow.protocol.OFExperimenter;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFFeaturesReply;
+import org.projectfloodlight.openflow.protocol.OFFlowModFailedCode;
+import org.projectfloodlight.openflow.protocol.OFFlowRemoved;
+import org.projectfloodlight.openflow.protocol.OFGetConfigReply;
+import org.projectfloodlight.openflow.protocol.OFGetConfigRequest;
+import org.projectfloodlight.openflow.protocol.OFHello;
+import org.projectfloodlight.openflow.protocol.OFHelloElem;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFNiciraControllerRole;
+import org.projectfloodlight.openflow.protocol.OFNiciraControllerRoleReply;
+import org.projectfloodlight.openflow.protocol.OFPacketIn;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFPortStatus;
+import org.projectfloodlight.openflow.protocol.OFQueueGetConfigReply;
+import org.projectfloodlight.openflow.protocol.OFRoleReply;
+import org.projectfloodlight.openflow.protocol.OFRoleRequest;
+import org.projectfloodlight.openflow.protocol.OFSetConfig;
+import org.projectfloodlight.openflow.protocol.OFStatsReply;
+import org.projectfloodlight.openflow.protocol.OFStatsReplyFlags;
+import org.projectfloodlight.openflow.protocol.OFStatsType;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.protocol.errormsg.OFBadRequestErrorMsg;
+import org.projectfloodlight.openflow.protocol.errormsg.OFFlowModFailedErrorMsg;
+import org.projectfloodlight.openflow.protocol.errormsg.OFRoleRequestFailedErrorMsg;
+import org.projectfloodlight.openflow.types.U32;
+import org.projectfloodlight.openflow.types.U64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Channel handler deals with the switch connection and dispatches
+ * switch messages to the appropriate locations.
+ * @author readams, gregor, saurav
+ */
+class OFChannelHandler extends IdleStateAwareChannelHandler {
+
+    private static final Logger log = LoggerFactory.getLogger(OFChannelHandler.class);
+
+    private static final long DEFAULT_ROLE_TIMEOUT_MS = 10*1000; // 10 sec
+    private final Controller controller;
+    private final Counters counters;
+    private IOFSwitch sw;
+    private long thisdpid; // channelHandler cached value of connected switch id
+    private Channel channel;
+    // State needs to be volatile because the HandshakeTimeoutHandler
+    // needs to check if the handshake is complete
+    private volatile ChannelState state;
+
+
+    // All role messaging is handled by the roleChanger. The channel state machine
+    // coordinates between the roleChanger and the controller-global-registry-service
+    // to determine controller roles per switch.
+    private RoleChanger roleChanger;
+    // Used to coordinate between the controller and the cleanup thread(?)
+    // for access to the global registry on a per switch basis.
+    volatile Boolean controlRequested;
+    // When a switch with a duplicate dpid is found (i.e we already have a
+    // connected switch with the same dpid), the new switch is immediately
+    // disconnected. At that point netty callsback channelDisconnected() which
+    // proceeds to cleaup switch state - we need to ensure that it does not cleanup
+    // switch state for the older (still connected) switch
+    private volatile Boolean duplicateDpidFound;
+
+    // Temporary storage for switch-features and port-description
+    private OFFeaturesReply featuresReply;
+    private OFPortDescStatsReply portDescReply;
+    // a concurrent ArrayList to temporarily store port status messages
+    // before we are ready to deal with them
+    private final CopyOnWriteArrayList<OFPortStatus> pendingPortStatusMsg;
+
+    //Indicates the openflow version used by this switch
+    protected OFVersion ofVersion;
+    protected static OFFactory factory13;
+    protected static OFFactory factory10;
+
+    /** transaction Ids to use during handshake. Since only one thread
+     * calls into an OFChannelHandler instance, we don't need atomic.
+     * We will count down
+     */
+    private int handshakeTransactionIds = -1;
+
+    /**
+     * Create a new unconnected OFChannelHandler.
+     * @param controller
+     */
+    OFChannelHandler(Controller controller) {
+        this.controller = controller;
+        this.counters = controller.getCounters();
+        this.roleChanger = new RoleChanger(DEFAULT_ROLE_TIMEOUT_MS);
+        this.state = ChannelState.INIT;
+        this.pendingPortStatusMsg = new CopyOnWriteArrayList<OFPortStatus>();
+        factory13 = controller.getOFMessageFactory_13();
+        factory10 = controller.getOFMessageFactory_10();
+        controlRequested = Boolean.FALSE;
+        duplicateDpidFound = Boolean.FALSE;
+    }
+
+    //*******************
+    //  Role Handling
+    //*******************
+
+
+    /**
+     * When we remove a pending role request we use this enum to indicate how we
+     * arrived at the decision. When we send a role request to the switch, we
+     * also use  this enum to indicate what we expect back from the switch, so the
+     * role changer can match the reply to our expectation.
+     * @author gregor, saurav
+     */
+    public enum RoleRecvStatus {
+        /** The switch returned an error indicating that roles are not
+         * supported*/
+        UNSUPPORTED,
+        /** The request timed out */
+        NO_REPLY,
+        /** The reply was old, there is a newer request pending */
+        OLD_REPLY,
+        /**
+         *  The reply's role matched the role that this controller set in the
+         *  request message - invoked either initially at startup or to reassert
+         *  current role
+         */
+        MATCHED_CURRENT_ROLE,
+        /**
+         *  The reply's role matched the role that this controller set in the
+         *  request message - this is the result of a callback from the
+         *  global registry, followed by a role request sent to the switch
+         */
+        MATCHED_SET_ROLE,
+        /**
+         * The reply's role was a response to the query made by this controller
+         */
+        REPLY_QUERY,
+        /** We received a role reply message from the switch
+         *  but the expectation was unclear, or there was no expectation
+         */
+        OTHER_EXPECTATION,
+    }
+
+    /**
+     * Forwards to RoleChanger. See there.
+     * @param role
+     */
+    public void sendRoleRequest(Role role, RoleRecvStatus expectation) {
+        try {
+            roleChanger.sendRoleRequest(role, expectation);
+        } catch (IOException e) {
+             log.error("Disconnecting switch {} due to IO Error: {}",
+                              getSwitchInfoString(), e.getMessage());
+             channel.close();
+        }
+    }
+
+    // XXX S consider if necessary
+    public void disconnectSwitch() {
+        sw.disconnectSwitch();
+    }
+
+    /**
+     * A utility class to handle role requests and replies for this channel.
+     * After a role request is submitted the role changer keeps track of the
+     * pending request, collects the reply (if any) and times out the request
+     * if necessary.
+     *
+     * To simplify role handling we only keep track of the /last/ pending
+     * role reply send to the switch. If multiple requests are pending and
+     * we receive replies for earlier requests we ignore them. However, this
+     * way of handling pending requests implies that we could wait forever if
+     * a new request is submitted before the timeout triggers. If necessary
+     * we could work around that though.
+     * @author gregor
+     * @author saurav (added support OF1.3 role messages, and expectations)
+     */
+    private class RoleChanger {
+        // indicates that a request is currently pending
+        // needs to be volatile to allow correct double-check idiom
+        private volatile boolean requestPending;
+        // the transaction Id of the pending request
+        private int pendingXid;
+        // the role that's pending
+        private Role pendingRole;
+        // system time in MS when we send the request
+        private long roleSubmitTime;
+        // the timeout to use
+        private final long roleTimeoutMs;
+        // the expectation set by the caller for the returned role
+        private RoleRecvStatus expectation;
+
+        public RoleChanger(long roleTimeoutMs) {
+            this.requestPending = false;
+            this.roleSubmitTime = 0;
+            this.pendingXid = -1;
+            this.pendingRole = null;
+            this.roleTimeoutMs = roleTimeoutMs;
+            this.expectation = RoleRecvStatus.MATCHED_CURRENT_ROLE;
+        }
+
+        /**
+         * Send NX role request message to the switch requesting the specified
+         * role.
+         *
+         * @param sw switch to send the role request message to
+         * @param role role to request
+         */
+        private int sendNxRoleRequest(Role role) throws IOException {
+            // Convert the role enum to the appropriate role to send
+            OFNiciraControllerRole roleToSend = OFNiciraControllerRole.ROLE_OTHER;
+            switch (role) {
+            case MASTER:
+                roleToSend = OFNiciraControllerRole.ROLE_MASTER;
+                break;
+            case SLAVE:
+            case EQUAL:
+            default:
+                // ensuring that the only two roles sent to 1.0 switches with
+                // Nicira role support, are MASTER and SLAVE
+                roleToSend = OFNiciraControllerRole.ROLE_SLAVE;
+                log.warn("Sending Nx Role.SLAVE to switch {}.", sw);
+            }
+            int xid = sw.getNextTransactionId();
+            OFExperimenter roleRequest = factory10
+                    .buildNiciraControllerRoleRequest()
+                    .setXid(xid)
+                    .setRole(roleToSend)
+                    .build();
+            sw.write(Collections.<OFMessage>singletonList(roleRequest),
+                     new FloodlightContext());
+            return xid;
+        }
+
+        private int sendOF13RoleRequest(Role role) throws IOException {
+            // Convert the role enum to the appropriate role to send
+            OFControllerRole roleToSend = OFControllerRole.ROLE_NOCHANGE;
+            switch (role) {
+            case EQUAL:
+                roleToSend = OFControllerRole.ROLE_EQUAL;
+                break;
+            case MASTER:
+                roleToSend = OFControllerRole.ROLE_MASTER;
+                break;
+            case SLAVE:
+                roleToSend = OFControllerRole.ROLE_SLAVE;
+                break;
+            default:
+                log.warn("Sending default role.noChange to switch {}."
+                        + " Should only be used for queries.", sw);
+            }
+
+            int xid = sw.getNextTransactionId();
+            OFRoleRequest rrm = factory13
+                    .buildRoleRequest()
+                    .setRole(roleToSend)
+                    .setXid(xid)
+                    .setGenerationId(sw.getNextGenerationId())
+                    .build();
+            sw.write(rrm, null);
+            return xid;
+        }
+
+        /**
+         * Send a role request with the given role to the switch and update
+         * the pending request and timestamp.
+         * Sends an OFPT_ROLE_REQUEST to an OF1.3 switch, OR
+         * Sends an NX_ROLE_REQUEST to an OF1.0 switch if configured to support it
+         * in the IOFSwitch driver. If not supported, this method sends nothing
+         * and returns 'false'. The caller should take appropriate action.
+         *
+         * One other optimization we do here is that for OF1.0 switches with
+         * Nicira role message support, we force the Role.EQUAL to become
+         * Role.SLAVE, as there is no defined behavior for the Nicira role OTHER.
+         * We cannot expect it to behave like SLAVE. We don't have this problem with
+         * OF1.3 switches, because Role.EQUAL is well defined and we can simulate
+         * SLAVE behavior by using ASYNC messages.
+         *
+         * @param role
+         * @throws IOException
+         * @returns false if and only if the switch does not support role-request
+         * messages, according to the switch driver; true otherwise.
+         */
+        synchronized boolean sendRoleRequest(Role role, RoleRecvStatus expectation)
+                throws IOException {
+            this.expectation = expectation;
+
+            if (ofVersion == OFVersion.OF_10) {
+                Boolean supportsNxRole = (Boolean)
+                        sw.getAttribute(IOFSwitch.SWITCH_SUPPORTS_NX_ROLE);
+                if (!supportsNxRole) {
+			log.debug("Switch driver indicates no support for Nicira "
+					+ "role request messages. Not sending ...");
+			state.handleUnsentRoleMessage(OFChannelHandler.this, role,
+					expectation);
+			return false;
+                }
+                // OF1.0 switch with support for NX_ROLE_REQUEST vendor extn.
+                // make Role.EQUAL become Role.SLAVE
+                role = (role == Role.EQUAL) ? Role.SLAVE : role;
+                pendingXid = sendNxRoleRequest(role);
+                pendingRole = role;
+                roleSubmitTime = System.currentTimeMillis();
+                requestPending = true;
+            } else {
+                // OF1.3 switch, use OFPT_ROLE_REQUEST message
+                pendingXid = sendOF13RoleRequest(role);
+                pendingRole = role;
+                roleSubmitTime = System.currentTimeMillis();
+                requestPending = true;
+            }
+            return true;
+        }
+
+        /**
+         * Deliver a received role reply.
+         *
+         * Check if a request is pending and if the received reply matches the
+         * the expected pending reply (we check both role and xid) we set
+         * the role for the switch/channel.
+         *
+         * If a request is pending but doesn't match the reply we ignore it, and
+         * return
+         *
+         * If no request is pending we disconnect with a SwitchStateException
+         *
+         * @param RoleReplyInfo information about role-reply in format that
+         *                      controller can understand.
+         * @throws SwitchStateException if no request is pending
+         */
+        synchronized RoleRecvStatus deliverRoleReply(RoleReplyInfo rri)
+                throws SwitchStateException {
+            if (!requestPending) {
+                Role currentRole = (sw != null) ? sw.getRole() : null;
+                if (currentRole != null) {
+                    if (currentRole == rri.getRole()) {
+                        // Don't disconnect if the role reply we received is
+                        // for the same role we are already in.
+                        log.debug("Received unexpected RoleReply from "
+                                + "Switch: {} in State: {}. "
+                                + "Role in reply is same as current role of this "
+                                + "controller for this sw. Ignoring ...",
+                                getSwitchInfoString(), state.toString());
+                        return RoleRecvStatus.OTHER_EXPECTATION;
+                    } else {
+                        String msg = String.format("Switch: [%s], State: [%s], "
+                                + "received unexpected RoleReply[%s]. "
+                                + "No roles are pending, and this controller's "
+                                + "current role:[%s] does not match reply. "
+                                + "Disconnecting switch ... ",
+                                OFChannelHandler.this.getSwitchInfoString(),
+                                OFChannelHandler.this.state.toString(),
+                                rri, currentRole);
+                        throw new SwitchStateException(msg);
+                    }
+                }
+                log.debug("Received unexpected RoleReply {} from "
+                        + "Switch: {} in State: {}. "
+                        + "This controller has no current role for this sw. "
+                        + "Ignoring ...", new Object[] {rri,
+                                getSwitchInfoString(), state});
+                return RoleRecvStatus.OTHER_EXPECTATION;
+            }
+
+            int xid = (int) rri.getXid();
+            Role role = rri.getRole();
+            // XXX S should check generation id meaningfully and other cases of expectations
+            // U64 genId = rri.getGenId();
+
+            if (pendingXid != xid) {
+                log.debug("Received older role reply from " +
+                        "switch {} ({}). Ignoring. " +
+                        "Waiting for {}, xid={}",
+                        new Object[] { getSwitchInfoString(), rri,
+                        pendingRole, pendingXid });
+                return RoleRecvStatus.OLD_REPLY;
+            }
+
+            if (pendingRole == role) {
+                log.debug("Received role reply message from {} that matched "
+                        + "expected role-reply {} with expectations {}",
+                        new Object[] {getSwitchInfoString(), role, expectation});
+                counters.roleReplyReceived.updateCounterWithFlush();
+                //setSwitchRole(role, RoleRecvStatus.RECEIVED_REPLY); dont want to set state here
+                if (expectation == RoleRecvStatus.MATCHED_CURRENT_ROLE ||
+                        expectation == RoleRecvStatus.MATCHED_SET_ROLE) {
+                    return expectation;
+                } else {
+                    return RoleRecvStatus.OTHER_EXPECTATION;
+                }
+            }
+
+            // if xids match but role's don't, perhaps its a query (OF1.3)
+            if (expectation == RoleRecvStatus.REPLY_QUERY)
+                return expectation;
+
+            return RoleRecvStatus.OTHER_EXPECTATION;
+        }
+
+        /**
+         * Called if we receive an  error message. If the xid matches the
+         * pending request we handle it otherwise we ignore it.
+         *
+         * Note: since we only keep the last pending request we might get
+         * error messages for earlier role requests that we won't be able
+         * to handle
+         */
+        synchronized RoleRecvStatus deliverError(OFErrorMsg error)
+                throws SwitchStateException {
+            if (!requestPending) {
+                log.debug("Received an error msg from sw {}, but no pending "
+                        + "requests in role-changer; not handling ...",
+                        getSwitchInfoString());
+                return RoleRecvStatus.OTHER_EXPECTATION;
+            }
+            if (pendingXid != error.getXid()) {
+                if (error.getErrType() == OFErrorType.ROLE_REQUEST_FAILED) {
+                    log.debug("Received an error msg from sw {} for a role request,"
+                            + " but not for pending request in role-changer; "
+                            + " ignoring error {} ...",
+                            getSwitchInfoString(), error);
+                }
+                return RoleRecvStatus.OTHER_EXPECTATION;
+            }
+            // it is an error related to a currently pending role request message
+            if (error.getErrType() == OFErrorType.BAD_REQUEST) {
+                counters.roleReplyErrorUnsupported.updateCounterWithFlush();
+                log.error("Received a error msg {} from sw {} in state {} for "
+                        + "pending role request {}. Switch driver indicates "
+                        + "role-messaging is supported. Possible issues in "
+                        + "switch driver configuration?", new Object[] {
+                                ((OFBadRequestErrorMsg)error).toString(),
+                                getSwitchInfoString(), state, pendingRole
+                        });
+                return RoleRecvStatus.UNSUPPORTED;
+            }
+
+            if (error.getErrType() == OFErrorType.ROLE_REQUEST_FAILED) {
+                OFRoleRequestFailedErrorMsg rrerr =
+                        (OFRoleRequestFailedErrorMsg) error;
+                switch (rrerr.getCode()) {
+                case BAD_ROLE:
+                    // switch says that current-role-req has bad role?
+                    // for now we disconnect
+                    // fall-thru
+                case STALE:
+                    // switch says that current-role-req has stale gen-id?
+                    // for now we disconnect
+                    // fall-thru
+                case UNSUP:
+                    // switch says that current-role-req has role that
+                    // cannot be supported? for now we disconnect
+                    String msgx = String.format("Switch: [%s], State: [%s], "
+                            + "received Error to for pending role request [%s]. "
+                            + "Error:[%s]. Disconnecting switch ... ",
+                            OFChannelHandler.this.getSwitchInfoString(),
+                            OFChannelHandler.this.state.toString(),
+                            pendingRole, rrerr);
+                    throw new SwitchStateException(msgx);
+                default:
+                    break;
+                }
+            }
+
+            // This error message was for a role request message but we dont know
+            // how to handle errors for nicira role request messages
+            return RoleRecvStatus.OTHER_EXPECTATION;
+        }
+
+        /**
+         * Check if a pending role request has timed out.
+         */
+        void checkTimeout() {
+            if (!requestPending)
+                return;
+            synchronized(this) {
+                if (!requestPending)
+                    return;
+                long now = System.currentTimeMillis();
+                if (now - roleSubmitTime > roleTimeoutMs) {
+                    // timeout triggered.
+                    counters.roleReplyTimeout.updateCounterWithFlush();
+                    //setSwitchRole(pendingRole, RoleRecvStatus.NO_REPLY);
+                    // XXX S come back to this
+                }
+            }
+        }
+
+    }
+
+    //*************************
+    //  Channel State Machine
+    //*************************
+
+    /**
+     * The state machine for handling the switch/channel state. All state
+     * transitions should happen from within the state machine (and not from other
+     * parts of the code)
+     * @author gregor
+     * @author saurav (modified to handle 1.0 & 1.3 switches, EQUAL state, role-handling )
+     */
+    enum ChannelState {
+        /**
+         * Initial state before channel is connected.
+         */
+        INIT(false) {
+            @Override
+            void processOFMessage(OFChannelHandler h, OFMessage m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException {
+                // need to implement since its abstract but it will never
+                // be called
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                unhandledMessageReceived(h, m);
+            }
+        },
+
+        /**
+         * We send a OF 1.3 HELLO to the switch and wait for a Hello from the switch.
+         * Once we receive the reply, we decide on OF 1.3 or 1.0 switch - no other
+         * protocol version is accepted.
+         * We send an OFFeaturesRequest depending on the protocol version selected
+         * Next state is WAIT_FEATURES_REPLY
+         */
+        WAIT_HELLO(false) {
+            @Override
+            void processOFHello(OFChannelHandler h, OFHello m)
+                    throws IOException {
+                // TODO We could check for the optional bitmap, but for now
+                // we are just checking the version number.
+                if (m.getVersion() == OFVersion.OF_13) {
+                    log.info("Received {} Hello from {}", m.getVersion(),
+                            h.channel.getRemoteAddress());
+                    h.ofVersion = OFVersion.OF_13;
+                } else if (m.getVersion() == OFVersion.OF_10) {
+                    log.info("Received {} Hello from {} - switching to OF "
+                            + "version 1.0", m.getVersion(),
+                            h.channel.getRemoteAddress());
+                    h.ofVersion = OFVersion.OF_10;
+                } else {
+                    log.error("Received Hello of version {} from switch at {}. "
+                            + "This controller works with OF1.0 and OF1.3 "
+                            + "switches. Disconnecting switch ...",
+                            m.getVersion(), h.channel.getRemoteAddress());
+                    h.channel.disconnect();
+                    return;
+                }
+                h.sendHandshakeFeaturesRequestMessage();
+                h.setState(WAIT_FEATURES_REPLY);
+            }
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                                          OFStatsReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m) {
+                logErrorDisconnect(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                unhandledMessageReceived(h, m);
+            }
+        },
+
+
+        /**
+         * We are waiting for a features reply message. Once we receive it, the
+         * behavior depends on whether this is a 1.0 or 1.3 switch. For 1.0,
+         * we send a SetConfig request, barrier, and GetConfig request and the
+         * next state is WAIT_CONFIG_REPLY. For 1.3, we send a Port description
+         * request and the next state is WAIT_PORT_DESC_REPLY.
+         */
+        WAIT_FEATURES_REPLY(false) {
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException {
+                h.thisdpid = m.getDatapathId().getLong();
+                log.info("Received features reply for switch at {} with dpid {}",
+                        h.getSwitchInfoString(), h.thisdpid);
+                //update the controller about this connected switch
+                boolean success = h.controller.addConnectedSwitch(
+                        h.thisdpid, h);
+                if (!success) {
+                    disconnectDuplicate(h);
+                    return;
+                }
+
+                h.featuresReply = m; //temp store
+                if (h.ofVersion == OFVersion.OF_10) {
+                    h.sendHandshakeSetConfig();
+                    h.setState(WAIT_CONFIG_REPLY);
+                } else {
+                    //version is 1.3, must get switchport information
+                    h.sendHandshakeOFPortDescRequest();
+                    h.setState(WAIT_PORT_DESC_REPLY);
+                }
+            }
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                                          OFStatsReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m) {
+                logErrorDisconnect(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                unhandledMessageReceived(h, m);
+            }
+        },
+
+        /**
+         * We are waiting for a description of the 1.3 switch ports.
+         * Once received, we send a SetConfig request
+         * Next State is WAIT_CONFIG_REPLY
+         */
+        WAIT_PORT_DESC_REPLY(false) {
+
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h, OFStatsReply m)
+                    throws SwitchStateException {
+                // Read port description
+                if (m.getStatsType() != OFStatsType.PORT_DESC) {
+                    log.warn("Expecting port description stats but received stats "
+                            + "type {} from {}. Ignoring ...", m.getStatsType(),
+                            h.channel.getRemoteAddress());
+                    return;
+                }
+                if (m.getFlags().contains(OFStatsReplyFlags.REPLY_MORE)) {
+                    log.warn("Stats reply indicates more stats from sw {} for "
+                            + "port description - not currently handled",
+                            h.getSwitchInfoString());
+                }
+                h.portDescReply = (OFPortDescStatsReply)m; // temp store
+                log.info("Received port desc reply for switch at {}",
+                        h.getSwitchInfoString());
+                try {
+                    h.sendHandshakeSetConfig();
+                } catch (IOException e) {
+                    log.error("Unable to send setConfig after PortDescReply. "
+                            + "Error: {}", e.getMessage());
+                }
+                h.setState(WAIT_CONFIG_REPLY);
+            }
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException, SwitchStateException {
+                logErrorDisconnect(h, m);
+
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                unhandledMessageReceived(h, m);
+
+            }
+        },
+
+        /**
+         * We are waiting for a config reply message. Once we receive it
+         * we send a DescriptionStatsRequest to the switch.
+         * Next state: WAIT_DESCRIPTION_STAT_REPLY
+         */
+        WAIT_CONFIG_REPLY(false) {
+            @Override
+            @LogMessageDocs({
+                @LogMessageDoc(level="WARN",
+                        message="Config Reply from {switch} has " +
+                                "miss length set to {length}",
+                        explanation="The controller requires that the switch " +
+                                "use a miss length of 0xffff for correct " +
+                                "function",
+                        recommendation="Use a different switch to ensure " +
+                                "correct function")
+            })
+            void processOFGetConfigReply(OFChannelHandler h, OFGetConfigReply m)
+                    throws IOException {
+                if (m.getMissSendLen() == 0xffff) {
+                    log.trace("Config Reply from switch {} confirms "
+                            + "miss length set to 0xffff",
+                            h.getSwitchInfoString());
+                } else {
+                    // FIXME: we can't really deal with switches that don't send
+                    // full packets. Shouldn't we drop the connection here?
+                    log.warn("Config Reply from switch {} has"
+                            + "miss length set to {}",
+                            h.getSwitchInfoString(),
+                            m.getMissSendLen());
+                }
+                h.sendHandshakeDescriptionStatsRequest();
+                h.setState(WAIT_DESCRIPTION_STAT_REPLY);
+            }
+
+            @Override
+            void processOFBarrierReply(OFChannelHandler h, OFBarrierReply m) {
+                // do nothing;
+            }
+
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                                          OFStatsReply  m)
+                    throws IOException, SwitchStateException {
+                log.error("Received multipart(stats) message sub-type {}",
+                        m.getStatsType());
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m) {
+                logErrorDisconnect(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                h.pendingPortStatusMsg.add(m);
+            }
+        },
+
+
+        /**
+         * We are waiting for a OFDescriptionStat message from the switch.
+         * Once we receive any stat message we try to parse it. If it's not
+         * a description stats message we disconnect. If its the expected
+         * description stats message, we:
+         *    - use the switch driver to bind the switch and get an IOFSwitch instance
+         *    - setup the IOFSwitch instance
+         *    - add switch to FloodlightProvider(Controller) and send the initial role
+         *      request to the switch.
+         * Next state: WAIT_INITIAL_ROLE
+         *      In the typical case, where switches support role request messages
+         *      the next state is where we expect the role reply message.
+         *      In the special case that where the switch does not support any kind
+         *      of role request messages, we don't send a role message, but we do
+         *      request mastership from the registry service. This controller
+         *      should become master once we hear back from the registry service.
+         * All following states will have a h.sw instance!
+         */
+        WAIT_DESCRIPTION_STAT_REPLY(false) {
+            @LogMessageDoc(message="Switch {switch info} bound to class " +
+                "{switch driver}, description {switch description}",
+                    explanation="The specified switch has been bound to " +
+                            "a switch driver based on the switch description" +
+                            "received from the switch")
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h, OFStatsReply m)
+                    throws SwitchStateException {
+                // Read description, if it has been updated
+                if (m.getStatsType() != OFStatsType.DESC) {
+                    log.warn("Expecting Description stats but received stats "
+                            + "type {} from {}. Ignoring ...", m.getStatsType(),
+                            h.channel.getRemoteAddress());
+                    return;
+                }
+                log.info("Received switch description reply from switch at {}",
+                        h.channel.getRemoteAddress());
+                OFDescStatsReply drep = (OFDescStatsReply) m;
+                // Here is where we differentiate between different kinds of switches
+                h.sw = h.controller.getOFSwitchInstance(drep, h.ofVersion);
+                // set switch information
+                h.sw.setOFVersion(h.ofVersion);
+                ((OFSwitchImplBase) h.sw).setFeaturesReply(h.featuresReply);
+                ((OFSwitchImplBase) h.sw).setPortDescReply(h.portDescReply);
+                h.sw.setConnected(true);
+                h.sw.setChannel(h.channel);
+                h.sw.setFloodlightProvider(h.controller);
+                h.sw.setThreadPoolService(h.controller.getThreadPoolService());
+                try {
+                    h.sw.setDebugCounterService(h.controller.getDebugCounter());
+                } catch (CounterException e) {
+                    h.counters.switchCounterRegistrationFailed
+                            .updateCounterNoFlush();
+                    log.warn("Could not register counters for switch {} ",
+                              h.getSwitchInfoString(), e);
+                }
+
+                log.info("Switch {} bound to class {}, description {}",
+                         new Object[] { h.sw, h.sw.getClass(), drep });
+                //Put switch in EQUAL mode until we hear back from the global registry
+                log.debug("Setting new switch {} to EQUAL and sending Role request",
+                        h.sw.getStringId());
+                h.setSwitchRole(Role.EQUAL);
+                try {
+                    boolean supportsRRMsg = h.roleChanger.sendRoleRequest(Role.EQUAL,
+                            RoleRecvStatus.MATCHED_CURRENT_ROLE);
+                    if (!supportsRRMsg) {
+                        log.warn("Switch {} does not support role request messages "
+                                + "of any kind. No role messages were sent. "
+                                + "This controller instance SHOULD become MASTER "
+                                + "from the registry process. ",
+                                h.getSwitchInfoString());
+                    }
+                    h.setState(WAIT_INITIAL_ROLE);
+                    // request control of switch from global registry -
+                    // necessary even if this is the only controller the
+                    // switch is connected to.
+                    h.controller.submitRegistryRequest(h.sw.getId());
+                } catch (IOException e) {
+                    log.error("Exception when sending role request: {} ",
+                            e.getMessage());
+                    // FIXME shouldn't we disconnect?
+                }
+            }
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m) {
+                logErrorDisconnect(h, m);
+            }
+
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException {
+                h.pendingPortStatusMsg.add(m);
+            }
+        },
+
+        /**
+         * We are waiting for a role reply message in response to a role request
+         * sent after hearing back from the registry service -- OR -- we are
+         * just waiting to hear back from the registry service in the case that
+         * the switch does not support role messages. If completed successfully,
+         * the controller's role for this switch will be set here.
+         * Before we move to the state corresponding to the role, we allow the
+         * switch specific driver to complete its configuration. This configuration
+         * typically depends on the role the controller is playing for this switch.
+         * And so we set the switch role (for 'this' controller) before we start
+         * the driver-sub-handshake.
+         * Next State: WAIT_SWITCH_DRIVER_SUB_HANDSHAKE
+         */
+        WAIT_INITIAL_ROLE(false) {
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws SwitchStateException {
+                // role changer will ignore the error if it isn't for it
+                RoleRecvStatus rrstatus = h.roleChanger.deliverError(m);
+                if (rrstatus == RoleRecvStatus.OTHER_EXPECTATION) {
+                    logError(h, m);
+                }
+            }
+
+            @Override
+            void processOFExperimenter(OFChannelHandler h, OFExperimenter m)
+                    throws IOException, SwitchStateException {
+                Role role = extractNiciraRoleReply(h, m);
+                // If role == null it means the vendor (experimenter) message
+                // wasn't really a Nicira role reply. We ignore this case.
+                if (role != null) {
+                    RoleReplyInfo rri = new RoleReplyInfo(role, null, m.getXid());
+                    RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(rri);
+                    if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+                        setRoleAndStartDriverHandshake(h, rri.getRole());
+                    } // else do nothing - wait for the correct expected reply
+                } else {
+                    unhandledMessageReceived(h, m);
+                }
+            }
+
+            @Override
+            void processOFRoleReply(OFChannelHandler h, OFRoleReply m)
+                    throws SwitchStateException, IOException {
+                RoleReplyInfo rri = extractOFRoleReply(h,m);
+                RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(rri);
+                if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+                    setRoleAndStartDriverHandshake(h, rri.getRole());
+                } // else do nothing - wait for the correct expected reply
+            }
+
+            @Override
+            void handleUnsentRoleMessage(OFChannelHandler h, Role role,
+            		RoleRecvStatus expectation) throws IOException {
+            	// typically this is triggered for a switch where role messages
+            	// are not supported - we confirm that the role being set is
+            	// master and move to the next state
+            	if (expectation == RoleRecvStatus.MATCHED_SET_ROLE) {
+            		if (role == Role.MASTER) {
+            			setRoleAndStartDriverHandshake(h, role);
+            		} else {
+            			log.error("Expected MASTER role from registry for switch "
+            					+ "which has no support for role-messages."
+            					+ "Received {}. It is possible that this switch "
+            					+ "is connected to other controllers, in which "
+            					+ "case it should support role messages - not "
+            					+ "moving forward.", role);
+            		}
+            	} // else do nothing - wait to hear back from registry
+
+            }
+
+            private void setRoleAndStartDriverHandshake(OFChannelHandler h,
+                    Role role) throws IOException {
+                h.setSwitchRole(role);
+                h.sw.startDriverHandshake();
+                if (h.sw.isDriverHandshakeComplete()) {
+                    Role mySwitchRole = h.sw.getRole();
+                    if (mySwitchRole == Role.MASTER) {
+                        log.info("Switch-driver sub-handshake complete. "
+                                + "Activating switch {} with Role: MASTER",
+                                h.getSwitchInfoString());
+                        handlePendingPortStatusMessages(h); //before activation
+                        boolean success = h.controller.addActivatedMasterSwitch(
+                                h.sw.getId(), h.sw);
+                        if (!success) {
+                            disconnectDuplicate(h);
+                            return;
+                        }
+                        h.setState(MASTER);
+                    } else {
+                        log.info("Switch-driver sub-handshake complete. "
+                                + "Activating switch {} with Role: EQUAL",
+                                h.getSwitchInfoString());
+                        handlePendingPortStatusMessages(h); //before activation
+                        boolean success = h.controller.addActivatedEqualSwitch(
+                                h.sw.getId(), h.sw);
+                        if (!success) {
+                            disconnectDuplicate(h);
+                            return;
+                        }
+                        h.setState(EQUAL);
+                    }
+                } else {
+                    h.setState(WAIT_SWITCH_DRIVER_SUB_HANDSHAKE);
+                }
+            }
+
+            @Override
+            void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                    throws IOException, SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h, OFStatsReply m)
+                    throws SwitchStateException {
+                illegalMessageReceived(h, m);
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                h.pendingPortStatusMsg.add(m);
+
+            }
+        },
+
+        /**
+         * We are waiting for the respective switch driver to complete its
+         * configuration. Notice that we do not consider this to be part of the main
+         * switch-controller handshake. But we do consider it as a step that comes
+         * before we declare the switch as available to the controller.
+         * Next State: depends on the role of this controller for this switch - either
+         * MASTER or EQUAL.
+         */
+        WAIT_SWITCH_DRIVER_SUB_HANDSHAKE(true) {
+
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException {
+                // will never be called. We override processOFMessage
+            }
+
+            @Override
+            void processOFMessage(OFChannelHandler h, OFMessage m)
+                    throws IOException {
+                if (m.getType() == OFType.ECHO_REQUEST)
+                    processOFEchoRequest(h, (OFEchoRequest)m);
+                else {
+                    // FIXME: other message to handle here?
+                    h.sw.processDriverHandshakeMessage(m);
+                    if (h.sw.isDriverHandshakeComplete()) {
+                        // consult the h.sw role and goto that state
+                        Role mySwitchRole = h.sw.getRole();
+                        if (mySwitchRole == Role.MASTER) {
+                            log.info("Switch-driver sub-handshake complete. "
+                                    + "Activating switch {} with Role: MASTER",
+                                    h.getSwitchInfoString());
+                            handlePendingPortStatusMessages(h); //before activation
+                            boolean success = h.controller.addActivatedMasterSwitch(
+                                    h.sw.getId(), h.sw);
+                            if (!success) {
+                                disconnectDuplicate(h);
+                                return;
+                            }
+                            h.setState(MASTER);
+                        } else {
+                            log.info("Switch-driver sub-handshake complete. "
+                                    + "Activating switch {} with Role: EQUAL",
+                                    h.getSwitchInfoString());
+                            handlePendingPortStatusMessages(h); //before activation
+                            boolean success = h.controller.addActivatedEqualSwitch(
+                                    h.sw.getId(), h.sw);
+                            if (!success) {
+                                disconnectDuplicate(h);
+                                return;
+                            }
+                            h.setState(EQUAL);
+                        }
+                    }
+                }
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                h.pendingPortStatusMsg.add(m);
+            }
+        },
+
+
+        /**
+         * This controller is in MASTER role for this switch. We enter this state
+         * after requesting and winning control from the global registry.
+         * The main handshake as well as the switch-driver sub-handshake
+         * is complete at this point.
+         * // XXX S reconsider below
+         * In the (near) future we may deterministically assign controllers to
+         * switches at startup.
+         * We only leave this state if the switch disconnects or
+         * if we send a role request for SLAVE /and/ receive the role reply for
+         * SLAVE.
+         */
+        MASTER(true) {
+            @LogMessageDoc(level="WARN",
+                message="Received permission error from switch {} while" +
+                         "being master. Reasserting master role.",
+                explanation="The switch has denied an operation likely " +
+                         "indicating inconsistent controller roles",
+                recommendation="This situation can occurs transiently during role" +
+                 " changes. If, however, the condition persists or happens" +
+                 " frequently this indicates a role inconsistency. " +
+                 LogMessageDoc.CHECK_CONTROLLER )
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException, SwitchStateException {
+                // first check if the error msg is in response to a role-request message
+                RoleRecvStatus rrstatus = h.roleChanger.deliverError(m);
+                if (rrstatus != RoleRecvStatus.OTHER_EXPECTATION) {
+			// rolechanger has handled the error message - we are done
+                    return;
+                }
+
+                // if we get here, then the error message is for something else
+                if (m.getErrType() == OFErrorType.BAD_REQUEST &&
+                        ((OFBadRequestErrorMsg) m).getCode() ==
+                            OFBadRequestCode.EPERM) {
+                    // We are the master controller and the switch returned
+                    // a permission error. This is a likely indicator that
+                    // the switch thinks we are slave. Reassert our
+                    // role
+                    // FIXME: this could be really bad during role transitions
+                    // if two controllers are master (even if its only for
+                    // a brief period). We might need to see if these errors
+                    // persist before we reassert
+                    h.counters.epermErrorWhileSwitchIsMaster.updateCounterWithFlush();
+                    log.warn("Received permission error from switch {} while" +
+                             "being master. Reasserting master role.",
+                             h.getSwitchInfoString());
+                    //h.controller.reassertRole(h, Role.MASTER);
+                    // XXX S reassert in role changer or reconsider if all this
+                    // stuff is really needed
+                } else if (m.getErrType() == OFErrorType.FLOW_MOD_FAILED &&
+                            ((OFFlowModFailedErrorMsg) m).getCode() ==
+                                OFFlowModFailedCode.ALL_TABLES_FULL) {
+                    h.sw.setTableFull(true);
+                } else {
+                    logError(h, m);
+                }
+                h.dispatchMessage(m);
+            }
+
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                                          OFStatsReply m) {
+                h.sw.deliverStatisticsReply(m);
+            }
+
+            @Override
+            void processOFExperimenter(OFChannelHandler h, OFExperimenter m)
+                    throws IOException, SwitchStateException {
+                Role role = extractNiciraRoleReply(h, m);
+                if (role == null) {
+			// The message wasn't really a Nicira role reply. We just
+			// dispatch it to the OFMessage listeners in this case.
+			h.dispatchMessage(m);
+			return;
+                }
+
+                RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(
+				new RoleReplyInfo(role, null, m.getXid()));
+                if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+			checkAndSetRoleTransition(h, role);
+                }
+            }
+
+            @Override
+            void processOFRoleReply(OFChannelHandler h, OFRoleReply m)
+                    throws SwitchStateException, IOException {
+                RoleReplyInfo rri = extractOFRoleReply(h, m);
+                RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(rri);
+                if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+			checkAndSetRoleTransition(h, rri.getRole());
+                }
+            }
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                handlePortStatusMessage(h, m, true);
+                h.dispatchMessage(m);
+            }
+
+            @Override
+            void processOFPacketIn(OFChannelHandler h, OFPacketIn m)
+                    throws IOException {
+                h.dispatchMessage(m);
+            }
+
+            @Override
+            void processOFFlowRemoved(OFChannelHandler h,
+                                      OFFlowRemoved m) throws IOException {
+                h.dispatchMessage(m);
+            }
+
+            @Override
+            void processOFBarrierReply(OFChannelHandler h, OFBarrierReply m)
+                    throws IOException {
+		h.dispatchMessage(m);
+            }
+
+        },
+
+        /**
+         * This controller is in EQUAL role for this switch. We enter this state
+         * after some /other/ controller instance wins mastership-role over this
+         * switch. The EQUAL role can be considered the same as the SLAVE role
+         * if this controller does NOT send commands or packets to the switch.
+         * This should always be true for OF1.0 switches. XXX S need to enforce.
+         *
+         * For OF1.3 switches, choosing this state as EQUAL instead of SLAVE,
+         * gives us the flexibility that if an app wants to send commands/packets
+         * to switches, it can, even thought it is running on a controller instance
+         * that is not in a MASTER role for this switch. Of course, it is the job
+         * of the app to ensure that commands/packets sent by this (EQUAL) controller
+         * instance does not clash/conflict with commands/packets sent by the MASTER
+         * controller for this switch. Neither the controller instances, nor the
+         * switch provides any kind of resolution mechanism should conflicts occur.
+         */
+        EQUAL(true) {
+            @Override
+            void processOFError(OFChannelHandler h, OFErrorMsg m)
+                    throws IOException, SwitchStateException {
+                // role changer will ignore the error if it isn't for it
+                RoleRecvStatus rrstatus = h.roleChanger.deliverError(m);
+                if (rrstatus == RoleRecvStatus.OTHER_EXPECTATION) {
+                    logError(h, m);
+                    h.dispatchMessage(m);
+                }
+            }
+
+            @Override
+            void processOFStatisticsReply(OFChannelHandler h,
+                                          OFStatsReply m) {
+                h.sw.deliverStatisticsReply(m);
+            }
+
+            @Override
+            void processOFExperimenter(OFChannelHandler h, OFExperimenter m)
+                    throws IOException, SwitchStateException {
+                Role role = extractNiciraRoleReply(h, m);
+                // If role == null it means the message wasn't really a
+                // Nicira role reply. We ignore it in this state.
+                if (role != null) {
+			RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(
+                            new RoleReplyInfo(role, null, m.getXid()));
+			if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+			checkAndSetRoleTransition(h, role);
+                    }
+                } else {
+                    unhandledMessageReceived(h, m);
+                }
+            }
+
+            @Override
+            void processOFRoleReply(OFChannelHandler h, OFRoleReply m)
+                    throws SwitchStateException, IOException {
+                RoleReplyInfo rri = extractOFRoleReply(h, m);
+                RoleRecvStatus rrs = h.roleChanger.deliverRoleReply(rri);
+                if (rrs == RoleRecvStatus.MATCHED_SET_ROLE) {
+			checkAndSetRoleTransition(h, rri.getRole());
+                }
+            }
+
+            // XXX S needs more handlers for 1.3 switches in equal role
+
+            @Override
+            void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                    throws IOException, SwitchStateException {
+                handlePortStatusMessage(h, m, true);
+            }
+
+            @Override
+            @LogMessageDoc(level="WARN",
+                message="Received PacketIn from switch {} while" +
+                         "being slave. Reasserting slave role.",
+                explanation="The switch has receive a PacketIn despite being " +
+                         "in slave role indicating inconsistent controller roles",
+                recommendation="This situation can occurs transiently during role" +
+                         " changes. If, however, the condition persists or happens" +
+                         " frequently this indicates a role inconsistency. " +
+                         LogMessageDoc.CHECK_CONTROLLER )
+            void processOFPacketIn(OFChannelHandler h, OFPacketIn m) throws IOException {
+                // we don't expect packetIn while slave, reassert we are slave
+                h.counters.packetInWhileSwitchIsSlave.updateCounterNoFlush();
+                log.warn("Received PacketIn from switch {} while" +
+                         "being slave. Reasserting slave role.", h.sw);
+                //h.controller.reassertRole(h, Role.SLAVE);
+                // XXX reassert in role changer
+            }
+        };
+
+        private final boolean handshakeComplete;
+        ChannelState(boolean handshakeComplete) {
+            this.handshakeComplete = handshakeComplete;
+        }
+
+        /**
+         * Is this a state in which the handshake has completed?
+         * @return true if the handshake is complete
+         */
+        public boolean isHandshakeComplete() {
+            return handshakeComplete;
+        }
+
+        /**
+         * Get a string specifying the switch connection, state, and
+         * message received. To be used as message for SwitchStateException
+         * or log messages
+         * @param h The channel handler (to get switch information_
+         * @param m The OFMessage that has just been received
+         * @param details A string giving more details about the exact nature
+         * of the problem.
+         * @return
+         */
+        // needs to be protected because enum members are actually subclasses
+        protected String getSwitchStateMessage(OFChannelHandler h,
+                                                      OFMessage m,
+                                                      String details) {
+            return String.format("Switch: [%s], State: [%s], received: [%s]"
+                                 + ", details: %s",
+                                 h.getSwitchInfoString(),
+                                 this.toString(),
+                                 m.getType().toString(),
+                                 details);
+        }
+
+        /**
+         * We have an OFMessage we didn't expect given the current state and
+         * we want to treat this as an error.
+         * We currently throw an exception that will terminate the connection
+         * However, we could be more forgiving
+         * @param h the channel handler that received the message
+         * @param m the message
+         * @throws SwitchStateException
+         * @throws SwitchStateExeption we always through the execption
+         */
+        // needs to be protected because enum members are acutally subclasses
+        protected void illegalMessageReceived(OFChannelHandler h, OFMessage m)
+                throws SwitchStateException {
+            String msg = getSwitchStateMessage(h, m,
+                    "Switch should never send this message in the current state");
+            throw new SwitchStateException(msg);
+
+        }
+
+        /**
+         * We have an OFMessage we didn't expect given the current state and
+         * we want to ignore the message
+         * @param h the channel handler the received the message
+         * @param m the message
+         */
+        protected void unhandledMessageReceived(OFChannelHandler h,
+                                                OFMessage m) {
+            h.counters.unhandledMessage.updateCounterNoFlush();
+            if (log.isDebugEnabled()) {
+                String msg = getSwitchStateMessage(h, m,
+                        "Ignoring unexpected message");
+                log.debug(msg);
+            }
+        }
+
+        /**
+         * Log an OpenFlow error message from a switch
+         * @param sw The switch that sent the error
+         * @param error The error message
+         */
+        @LogMessageDoc(level="ERROR",
+                message="Error {error type} {error code} from {switch} " +
+                        "in state {state}",
+                explanation="The switch responded with an unexpected error" +
+                        "to an OpenFlow message from the controller",
+                recommendation="This could indicate improper network operation. " +
+                        "If the problem persists restarting the switch and " +
+                        "controller may help."
+                )
+        protected void logError(OFChannelHandler h, OFErrorMsg error) {
+            log.error("{} from switch {} in state {}",
+                      new Object[] {
+                          error,
+                          h.getSwitchInfoString(),
+                          this.toString()});
+        }
+
+        /**
+         * Log an OpenFlow error message from a switch and disconnect the
+         * channel
+         * @param sw The switch that sent the error
+         * @param error The error message
+         */
+        protected void logErrorDisconnect(OFChannelHandler h, OFErrorMsg error) {
+            logError(h, error);
+            h.channel.disconnect();
+        }
+
+        /**
+         * log an error message for a duplicate dpid and disconnect this channel
+         */
+        protected void disconnectDuplicate(OFChannelHandler h) {
+            log.error("Duplicated dpid or incompleted cleanup - "
+                    + "disconnecting channel {}", h.getSwitchInfoString());
+            h.duplicateDpidFound = Boolean.TRUE;
+            h.channel.disconnect();
+        }
+
+        /**
+         * Extract the role from an OFVendor message.
+         *
+         * Extract the role from an OFVendor message if the message is a
+         * Nicira role reply. Otherwise return null.
+         *
+         * @param h The channel handler receiving the message
+         * @param vendorMessage The vendor message to parse.
+         * @return The role in the message if the message is a Nicira role
+         * reply, null otherwise.
+         * @throws SwitchStateException If the message is a Nicira role reply
+         * but the numeric role value is unknown.
+         */
+        protected Role extractNiciraRoleReply(OFChannelHandler h,
+			OFExperimenter experimenterMsg) throws SwitchStateException {
+		int vendor = (int) experimenterMsg.getExperimenter();
+		if (vendor != 0x2320) // magic number representing nicira
+			return null;
+		OFNiciraControllerRoleReply nrr =
+				(OFNiciraControllerRoleReply) experimenterMsg;
+
+		Role role = null;
+		OFNiciraControllerRole ncr = nrr.getRole();
+		switch(ncr) {
+		case ROLE_MASTER:
+			role = Role.MASTER;
+			break;
+		case ROLE_OTHER:
+			role = Role.EQUAL;
+			break;
+		case ROLE_SLAVE:
+			role = Role.SLAVE;
+			break;
+		default: //handled below
+		}
+
+		if (role == null) {
+			String msg = String.format("Switch: [%s], State: [%s], "
+					+ "received NX_ROLE_REPLY with invalid role "
+					+ "value %d",
+					h.getSwitchInfoString(),
+					this.toString(),
+					nrr.getRole());
+			throw new SwitchStateException(msg);
+		}
+		return role;
+        }
+
+        /**
+         * Helper class returns role reply information in the format understood
+         * by the controller.
+         */
+        protected class RoleReplyInfo {
+            private Role role;
+            private U64 genId;
+            private long xid;
+
+            RoleReplyInfo (Role role, U64 genId, long xid) {
+                this.role = role;
+                this.genId = genId;
+                this.xid = xid;
+            }
+            public Role getRole() { return role; }
+            public U64 getGenId() { return genId; }
+            public long getXid() { return xid; }
+            @Override
+            public String toString() {
+                return "[Role:" + role + " GenId:" + genId + " Xid:" + xid + "]";
+            }
+        }
+
+        /**
+         * Extract the role information from an OF1.3 Role Reply Message
+         * @param h
+         * @param rrmsg
+         * @return RoleReplyInfo object
+         * @throws SwitchStateException
+         */
+        protected RoleReplyInfo extractOFRoleReply(OFChannelHandler h,
+                OFRoleReply rrmsg) throws SwitchStateException {
+            OFControllerRole cr = rrmsg.getRole();
+            Role role = null;
+            switch(cr) {
+            case ROLE_EQUAL:
+                role = Role.EQUAL;
+                break;
+            case ROLE_MASTER:
+                role = Role.MASTER;
+                break;
+            case ROLE_SLAVE:
+                role = Role.SLAVE;
+                break;
+            case ROLE_NOCHANGE: // switch should send current role
+            default:
+                String msg = String.format("Unknown controller role {} "
+                        + "received from switch {}", cr, h.sw);
+                throw new SwitchStateException(msg);
+            }
+
+            return new RoleReplyInfo(role, rrmsg.getGenerationId(), rrmsg.getXid());
+        }
+
+        /**
+         * Handles all pending port status messages before a switch is declared
+         * activated in MASTER or EQUAL role. Note that since this handling
+         * precedes the activation (and therefore notification to IOFSwitchListerners)
+         * the changes to ports will already be visible once the switch is
+         * activated. As a result, no notifications are sent out for these
+         * pending portStatus messages.
+         * @param h
+         * @throws SwitchStateException
+         */
+        protected void handlePendingPortStatusMessages(OFChannelHandler h) {
+            try {
+                handlePendingPortStatusMessages(h, 0);
+            } catch (SwitchStateException e) {
+                // do nothing - exception msg printed
+            }
+        }
+
+        private void handlePendingPortStatusMessages(OFChannelHandler h, int index)
+                throws SwitchStateException {
+            if (h.sw == null) {
+                String msg = "State machine error: switch is null. Should never " +
+                        "happen";
+                throw new SwitchStateException(msg);
+            }
+            ArrayList<OFPortStatus> temp  = new ArrayList<OFPortStatus>();
+            for (OFPortStatus ps: h.pendingPortStatusMsg) {
+                temp.add(ps);
+                handlePortStatusMessage(h, ps, false);
+            }
+            temp.clear();
+            // expensive but ok - we don't expect too many port-status messages
+            // note that we cannot use clear(), because of the reasons below
+            h.pendingPortStatusMsg.removeAll(temp);
+            // the iterator above takes a snapshot of the list - so while we were
+            // dealing with the pending port-status messages, we could have received
+            // newer ones. Handle them recursively, but break the recursion after
+            // five steps to avoid an attack.
+            if (!h.pendingPortStatusMsg.isEmpty() && ++index < 5) {
+                handlePendingPortStatusMessages(h, index);
+            }
+        }
+
+        /**
+         * Handle a port status message.
+         *
+         * Handle a port status message by updating the port maps in the
+         * IOFSwitch instance and notifying Controller about the change so
+         * it can dispatch a switch update.
+         *
+         * @param h The OFChannelHhandler that received the message
+         * @param m The PortStatus message we received
+         * @param doNotify if true switch port changed events will be
+         * dispatched
+         * @throws SwitchStateException
+         *
+         */
+        protected void handlePortStatusMessage(OFChannelHandler h, OFPortStatus m,
+                boolean doNotify) throws SwitchStateException {
+            if (h.sw == null) {
+                String msg = getSwitchStateMessage(h, m,
+                        "State machine error: switch is null. Should never " +
+                        "happen");
+                throw new SwitchStateException(msg);
+            }
+
+            Collection<PortChangeEvent> changes = h.sw.processOFPortStatus(m);
+            if (doNotify) {
+                for (PortChangeEvent ev: changes)
+                    h.controller.notifyPortChanged(h.sw.getId(), ev.port, ev.type);
+            }
+        }
+
+        /**
+         * Checks if the role received (from the role-reply msg) is different
+         * from the existing role in the IOFSwitch object for this controller.
+         * If so, it transitions the controller to the new role. Note that
+         * the caller should have already verified that the role-reply msg
+         * received was in response to a role-request msg sent out by this
+         * controller after hearing from the registry service.
+         *
+         * @param h the ChannelHandler that received the message
+         * @param role the role in the recieved role reply message
+         */
+        protected void checkAndSetRoleTransition(OFChannelHandler h, Role role) {
+		// we received a role-reply in response to a role message
+		// sent after hearing from the registry service. It is
+		// possible that the role of this controller instance for
+		// this switch has changed:
+		// for 1.0 switch: from MASTER to SLAVE
+		// for 1.3 switch: from MASTER to EQUAL
+		if ((h.sw.getRole() == Role.MASTER && role == Role.SLAVE) ||
+			(h.sw.getRole() == Role.MASTER && role == Role.EQUAL)) {
+			// the mastership has changed
+			h.sw.setRole(role);
+			h.setState(EQUAL);
+			h.controller.transitionToEqualSwitch(h.sw.getId());
+			return;
+		}
+
+		// or for both 1.0 and 1.3 switches from EQUAL to MASTER.
+		// note that for 1.0, even though we mean SLAVE,
+		// internally we call the role EQUAL.
+		if (h.sw.getRole() == Role.EQUAL && role == Role.MASTER) {
+			// the mastership has changed
+			h.sw.setRole(role);
+			h.setState(MASTER);
+			h.controller.transitionToMasterSwitch(h.sw.getId());
+			return;
+		}
+        }
+
+        /**
+         * Process an OF message received on the channel and
+         * update state accordingly.
+         *
+         * The main "event" of the state machine. Process the received message,
+         * send follow up message if required and update state if required.
+         *
+         * Switches on the message type and calls more specific event handlers
+         * for each individual OF message type. If we receive a message that
+         * is supposed to be sent from a controller to a switch we throw
+         * a SwitchStateExeption.
+         *
+         * The more specific handlers can also throw SwitchStateExceptions
+         *
+         * @param h The OFChannelHandler that received the message
+         * @param m The message we received.
+         * @throws SwitchStateException
+         * @throws IOException
+         */
+        void processOFMessage(OFChannelHandler h, OFMessage m)
+                throws IOException, SwitchStateException {
+            h.roleChanger.checkTimeout();
+            switch(m.getType()) {
+                case HELLO:
+                    processOFHello(h, (OFHello)m);
+                    break;
+                case BARRIER_REPLY:
+                    processOFBarrierReply(h, (OFBarrierReply)m);
+                    break;
+                case ECHO_REPLY:
+                    processOFEchoReply(h, (OFEchoReply)m);
+                    break;
+                case ECHO_REQUEST:
+                    processOFEchoRequest(h, (OFEchoRequest)m);
+                    break;
+                case ERROR:
+                    processOFError(h, (OFErrorMsg)m);
+                    break;
+                case FEATURES_REPLY:
+                    processOFFeaturesReply(h, (OFFeaturesReply)m);
+                    break;
+                case FLOW_REMOVED:
+                    processOFFlowRemoved(h, (OFFlowRemoved)m);
+                    break;
+                case GET_CONFIG_REPLY:
+                    processOFGetConfigReply(h, (OFGetConfigReply)m);
+                    break;
+                case PACKET_IN:
+                    processOFPacketIn(h, (OFPacketIn)m);
+                    break;
+                case PORT_STATUS:
+                    processOFPortStatus(h, (OFPortStatus)m);
+                    break;
+                case QUEUE_GET_CONFIG_REPLY:
+                    processOFQueueGetConfigReply(h, (OFQueueGetConfigReply)m);
+                    break;
+                case STATS_REPLY: // multipart_reply in 1.3
+                    processOFStatisticsReply(h, (OFStatsReply)m);
+                    break;
+                case EXPERIMENTER:
+                    processOFExperimenter(h, (OFExperimenter)m);
+                    break;
+                case ROLE_REPLY:
+                    processOFRoleReply(h, (OFRoleReply)m);
+                    break;
+                case GET_ASYNC_REPLY:
+                    processOFGetAsyncReply(h, (OFAsyncGetReply)m);
+                    break;
+
+                // The following messages are sent to switches. The controller
+                // should never receive them
+                case SET_CONFIG:
+                case GET_CONFIG_REQUEST:
+                case PACKET_OUT:
+                case PORT_MOD:
+                case QUEUE_GET_CONFIG_REQUEST:
+                case BARRIER_REQUEST:
+                case STATS_REQUEST: // multipart request in 1.3
+                case FEATURES_REQUEST:
+                case FLOW_MOD:
+                case GROUP_MOD:
+                case TABLE_MOD:
+                case GET_ASYNC_REQUEST:
+                case SET_ASYNC:
+                case METER_MOD:
+                default:
+                    illegalMessageReceived(h, m);
+                    break;
+            }
+        }
+
+        /*-----------------------------------------------------------------
+         * Default implementation for message handlers in any state.
+         *
+         * Individual states must override these if they want a behavior
+         * that differs from the default.
+         *
+         * In general, these handlers simply ignore the message and do
+         * nothing.
+         *
+         * There are some exceptions though, since some messages really
+         * are handled the same way in every state (e.g., ECHO_REQUST) or
+         * that are only valid in a single state (e.g., HELLO, GET_CONFIG_REPLY
+         -----------------------------------------------------------------*/
+
+        void processOFHello(OFChannelHandler h, OFHello m)
+                throws IOException, SwitchStateException {
+            // we only expect hello in the WAIT_HELLO state
+            illegalMessageReceived(h, m);
+        }
+
+        void processOFBarrierReply(OFChannelHandler h, OFBarrierReply m)
+                throws IOException {
+            // Silently ignore.
+        }
+
+        void processOFEchoRequest(OFChannelHandler h, OFEchoRequest m)
+            throws IOException {
+            if (h.ofVersion == null) {
+                log.error("No OF version set for {}. Not sending Echo REPLY",
+                        h.channel.getRemoteAddress());
+                return;
+            }
+            OFFactory factory = (h.ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+            OFEchoReply reply = factory
+                    .buildEchoReply()
+                    .setXid(m.getXid())
+                    .setData(m.getData())
+                    .build();
+            h.channel.write(Collections.singletonList(reply));
+        }
+
+        void processOFEchoReply(OFChannelHandler h, OFEchoReply m)
+            throws IOException {
+            // Do nothing with EchoReplies !!
+        }
+
+        // no default implementation for OFError
+        // every state must override it
+        abstract void processOFError(OFChannelHandler h, OFErrorMsg m)
+                throws IOException, SwitchStateException;
+
+
+        void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
+                throws IOException, SwitchStateException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFFlowRemoved(OFChannelHandler h, OFFlowRemoved m)
+            throws IOException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFGetConfigReply(OFChannelHandler h, OFGetConfigReply m)
+                throws IOException, SwitchStateException {
+            // we only expect config replies in the WAIT_CONFIG_REPLY state
+            illegalMessageReceived(h, m);
+        }
+
+        void processOFPacketIn(OFChannelHandler h, OFPacketIn m)
+                throws IOException {
+            unhandledMessageReceived(h, m);
+        }
+
+        // no default implementation. Every state needs to handle it.
+        abstract void processOFPortStatus(OFChannelHandler h, OFPortStatus m)
+                throws IOException, SwitchStateException;
+
+        void processOFQueueGetConfigReply(OFChannelHandler h,
+                                          OFQueueGetConfigReply m)
+                throws IOException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFStatisticsReply(OFChannelHandler h, OFStatsReply m)
+                throws IOException, SwitchStateException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFExperimenter(OFChannelHandler h, OFExperimenter m)
+                throws IOException, SwitchStateException {
+            // TODO: it might make sense to parse the vendor message here
+            // into the known vendor messages we support and then call more
+            // specific event handlers
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFRoleReply(OFChannelHandler h, OFRoleReply m)
+                throws SwitchStateException, IOException {
+            unhandledMessageReceived(h, m);
+        }
+
+        void processOFGetAsyncReply(OFChannelHandler h,
+                OFAsyncGetReply m) {
+            unhandledMessageReceived(h, m);
+        }
+
+        void handleUnsentRoleMessage(OFChannelHandler h, Role role,
+			RoleRecvStatus expectation) throws IOException {
+		// do nothing in most states
+        }
+    }
+
+
+
+    //*************************
+    //  Channel handler methods
+    //*************************
+
+    @Override
+    @LogMessageDoc(message="New switch connection from {ip address}",
+                   explanation="A new switch has connected from the " +
+                            "specified IP address")
+    public void channelConnected(ChannelHandlerContext ctx,
+                                 ChannelStateEvent e) throws Exception {
+        counters.switchConnected.updateCounterWithFlush();
+        channel = e.getChannel();
+        log.info("New switch connection from {}",
+                 channel.getRemoteAddress());
+        sendHandshakeHelloMessage();
+        setState(ChannelState.WAIT_HELLO);
+    }
+
+    @Override
+    @LogMessageDoc(message="Disconnected switch {switch information}",
+                   explanation="The specified switch has disconnected.")
+    public void channelDisconnected(ChannelHandlerContext ctx,
+                                    ChannelStateEvent e) throws Exception {
+        log.info("Switch disconnected callback for sw:{}. Cleaning up ...",
+        		getSwitchInfoString());
+        if (thisdpid != 0) {
+        	if (duplicateDpidFound != Boolean.TRUE) {
+        		// if the disconnected switch (on this ChannelHandler)
+        		// was not one with a duplicate-dpid, it is safe to remove all
+        		// state for it at the controller. Notice that if the disconnected
+        		// switch was a duplicate-dpid, calling the method below would clear
+        		// all state for the original switch (with the same dpid),
+        		// which we obviously don't want.
+        		controller.removeConnectedSwitch(thisdpid);
+        	} else {
+        		// A duplicate was disconnected on this ChannelHandler,
+        		// this is the same switch reconnecting, but the original state was
+        		// not cleaned up - XXX check liveness of original ChannelHandler
+        		duplicateDpidFound = Boolean.FALSE;
+        	}
+        } else {
+        	log.warn("no dpid in channelHandler registered for "
+        			+ "disconnected switch {}", getSwitchInfoString());
+        }
+    }
+
+    @Override
+    @LogMessageDocs({
+        @LogMessageDoc(level="ERROR",
+                message="Disconnecting switch {switch} due to read timeout",
+                explanation="The connected switch has failed to send any " +
+                            "messages or respond to echo requests",
+                recommendation=LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level="ERROR",
+                message="Disconnecting switch {switch}: failed to " +
+                        "complete handshake",
+                explanation="The switch did not respond correctly " +
+                            "to handshake messages",
+                recommendation=LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level="ERROR",
+                message="Disconnecting switch {switch} due to IO Error: {}",
+                explanation="There was an error communicating with the switch",
+                recommendation=LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level="ERROR",
+                message="Disconnecting switch {switch} due to switch " +
+                        "state error: {error}",
+                explanation="The switch sent an unexpected message",
+                recommendation=LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level="ERROR",
+                message="Disconnecting switch {switch} due to " +
+                        "message parse failure",
+                explanation="Could not parse a message from the switch",
+                recommendation=LogMessageDoc.CHECK_SWITCH),
+        @LogMessageDoc(level="ERROR",
+                message="Terminating controller due to storage exception",
+                explanation=Controller.ERROR_DATABASE,
+                recommendation=LogMessageDoc.CHECK_CONTROLLER),
+        @LogMessageDoc(level="ERROR",
+                message="Could not process message: queue full",
+                explanation="OpenFlow messages are arriving faster than " +
+                            " the controller can process them.",
+                recommendation=LogMessageDoc.CHECK_CONTROLLER),
+        @LogMessageDoc(level="ERROR",
+                message="Error while processing message " +
+                        "from switch {switch} {cause}",
+                explanation="An error occurred processing the switch message",
+                recommendation=LogMessageDoc.GENERIC_ACTION)
+    })
+    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e)
+            throws Exception {
+        if (e.getCause() instanceof ReadTimeoutException) {
+            // switch timeout
+            log.error("Disconnecting switch {} due to read timeout",
+                                 getSwitchInfoString());
+            counters.switchDisconnectReadTimeout.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof HandshakeTimeoutException) {
+            log.error("Disconnecting switch {}: failed to complete handshake",
+                      getSwitchInfoString());
+            counters.switchDisconnectHandshakeTimeout.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof ClosedChannelException) {
+            log.debug("Channel for sw {} already closed", getSwitchInfoString());
+        } else if (e.getCause() instanceof IOException) {
+            log.error("Disconnecting switch {} due to IO Error: {}",
+                      getSwitchInfoString(), e.getCause().getMessage());
+            if (log.isDebugEnabled()) {
+                // still print stack trace if debug is enabled
+                log.debug("StackTrace for previous Exception: ", e.getCause());
+            }
+            counters.switchDisconnectIOError.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof SwitchStateException) {
+            log.error("Disconnecting switch {} due to switch state error: {}",
+                      getSwitchInfoString(), e.getCause().getMessage());
+            if (log.isDebugEnabled()) {
+                // still print stack trace if debug is enabled
+                log.debug("StackTrace for previous Exception: ", e.getCause());
+            }
+            counters.switchDisconnectSwitchStateException.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof OFParseError) {
+            log.error("Disconnecting switch "
+                                 + getSwitchInfoString() +
+                                 " due to message parse failure",
+                                 e.getCause());
+            counters.switchDisconnectParseError.updateCounterWithFlush();
+            ctx.getChannel().close();
+        } else if (e.getCause() instanceof RejectedExecutionException) {
+            log.warn("Could not process message: queue full");
+            counters.rejectedExecutionException.updateCounterWithFlush();
+        } else {
+            log.error("Error while processing message from switch "
+                                 + getSwitchInfoString()
+                                 + "state " + this.state, e.getCause());
+            counters.switchDisconnectOtherException.updateCounterWithFlush();
+            ctx.getChannel().close();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return getSwitchInfoString();
+    }
+
+    @Override
+    public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e)
+            throws Exception {
+        OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+        OFMessage m = factory.buildEchoRequest().build();
+        log.info("Sending Echo Request on idle channel: {}",
+                e.getChannel().getPipeline().getLast().toString());
+        e.getChannel().write(Collections.singletonList(m));
+        // XXX S some problems here -- echo request has no transaction id, and
+        // echo reply is not correlated to the echo request.
+    }
+
+    @Override
+    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e)
+            throws Exception {
+        if (e.getMessage() instanceof List) {
+            @SuppressWarnings("unchecked")
+            List<OFMessage> msglist = (List<OFMessage>)e.getMessage();
+
+            LoadMonitor.LoadLevel loadlevel;
+            int packets_dropped = 0;
+            int packets_allowed = 0;
+            int lldps_allowed = 0;
+
+            if (this.controller.overload_drop) {
+                loadlevel = this.controller.loadmonitor.getLoadLevel();
+            }
+            else {
+                loadlevel = LoadMonitor.LoadLevel.OK;
+            }
+
+            for (OFMessage ofm : msglist) {
+                counters.messageReceived.updateCounterNoFlush();
+                // Per-switch input throttling - placeholder for future throttling
+                /*if (sw != null && sw.inputThrottled(ofm)) {
+                    counters.messageInputThrottled.updateCounterNoFlush();
+                    continue;
+                }*/
+                try {
+                    if (this.controller.overload_drop &&
+                        !loadlevel.equals(LoadMonitor.LoadLevel.OK)) {
+                        switch (ofm.getType()) {
+                        case PACKET_IN:
+                            switch (loadlevel) {
+                            case VERYHIGH:
+                                // Drop all packet-ins, including LLDP/BDDPs
+                                packets_dropped++;
+                                continue;
+                            case HIGH:
+                                // Drop all packet-ins, except LLDP/BDDPs
+                                byte[] data = ((OFPacketIn)ofm).getData();
+                                if (data.length > 14) {
+                                    if (((data[12] == (byte)0x88) &&
+                                         (data[13] == (byte)0xcc)) ||
+                                        ((data[12] == (byte)0x89) &&
+                                         (data[13] == (byte)0x42))) {
+                                        lldps_allowed++;
+                                        packets_allowed++;
+                                        break;
+                                    }
+                                }
+                                packets_dropped++;
+                                continue;
+                            default:
+                                // Load not high, go ahead and process msg
+                                packets_allowed++;
+                                break;
+                            }
+                            break;
+                        default:
+                            // Process all non-packet-ins
+                            packets_allowed++;
+                            break;
+                        }
+                    }
+
+                    // Do the actual packet processing
+                    state.processOFMessage(this, ofm);
+
+                }
+                catch (Exception ex) {
+                    // We are the last handler in the stream, so run the
+                    // exception through the channel again by passing in
+                    // ctx.getChannel().
+                    Channels.fireExceptionCaught(ctx.getChannel(), ex);
+                }
+            }
+
+            if (loadlevel != LoadMonitor.LoadLevel.OK) {
+                if (log.isDebugEnabled()) {
+                    log.debug(
+                        "Overload: Detected {}, packets dropped={}",
+                        loadlevel.toString(), packets_dropped);
+                    log.debug(
+                        "Overload: Packets allowed={} (LLDP/BDDPs allowed={})",
+                        packets_allowed, lldps_allowed);
+                }
+            }
+        }
+        else {
+            //Channels.fireExceptionCaught(ctx.getChannel(),
+            //      new AssertionError("Message received from Channel is not a list"));
+            //TODO: Pankaj: move the counters using ONOS metrics implementation
+
+            counters.messageReceived.updateCounterNoFlush();
+            state.processOFMessage(this, (OFMessage) e.getMessage());
+        }
+
+        // Flush all thread local queues etc. generated by this train
+        // of messages.
+        this.controller.flushAll();
+    }
+
+
+
+    //*************************
+    //  Channel utility methods
+    //*************************
+
+    /**
+     * Is this a state in which the handshake has completed?
+     * @return true if the handshake is complete
+     */
+    public boolean isHandshakeComplete() {
+        return this.state.isHandshakeComplete();
+    }
+
+    private void dispatchMessage(OFMessage m) throws IOException {
+        // handleMessage will count
+        this.controller.handleMessage(this.sw, m, null);
+    }
+
+    /**
+     * Return a string describing this switch based on the already available
+     * information (DPID and/or remote socket)
+     * @return
+     */
+    private String getSwitchInfoString() {
+        if (sw != null)
+            return sw.toString();
+        String channelString;
+        if (channel == null || channel.getRemoteAddress() == null) {
+            channelString = "?";
+        } else {
+            channelString = channel.getRemoteAddress().toString();
+        }
+        String dpidString;
+        if (featuresReply == null) {
+            dpidString = "?";
+        } else {
+            dpidString = featuresReply.getDatapathId().toString();
+        }
+        return String.format("[%s DPID[%s]]", channelString, dpidString);
+    }
+
+    /**
+     * Update the channels state. Only called from the state machine.
+     * TODO: enforce restricted state transitions
+     * @param state
+     */
+    private void setState(ChannelState state) {
+        this.state = state;
+    }
+
+    /**
+     * Send hello message to the switch using the handshake transactions ids.
+     * @throws IOException
+     */
+    private void sendHandshakeHelloMessage() throws IOException {
+        // The OF protocol requires us to start things off by sending the highest
+        // version of the protocol supported.
+
+        // bitmap represents OF1.0 (ofp_version=0x01) and OF1.3 (ofp_version=0x04)
+        // see Sec. 7.5.1 of the OF1.3.4 spec
+        U32 bitmap = U32.ofRaw(0x00000012);
+        OFHelloElem hem = factory13.buildHelloElemVersionbitmap()
+                .setBitmaps(Collections.singletonList(bitmap))
+                .build();
+        OFMessage.Builder mb = factory13.buildHello()
+                .setXid(this.handshakeTransactionIds--)
+                .setElements(Collections.singletonList(hem));
+        log.info("Sending OF_13 Hello to {}", channel.getRemoteAddress());
+        channel.write(Collections.singletonList(mb.build()));
+    }
+
+    /**
+     * Send featuresRequest msg to the switch using the handshake transactions ids.
+     * @throws IOException
+     */
+    private void sendHandshakeFeaturesRequestMessage() throws IOException {
+        OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+        OFMessage m = factory.buildFeaturesRequest()
+                .setXid(this.handshakeTransactionIds--)
+                .build();
+        channel.write(Collections.singletonList(m));
+    }
+
+    private void setSwitchRole(Role role) {
+        sw.setRole(role);
+    }
+
+    /**
+     * Send the configuration requests to tell the switch we want full
+     * packets
+     * @throws IOException
+     */
+    private void sendHandshakeSetConfig() throws IOException {
+        OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+        //log.debug("Sending CONFIG_REQUEST to {}", channel.getRemoteAddress());
+        List<OFMessage> msglist = new ArrayList<OFMessage>(3);
+
+        // Ensure we receive the full packet via PacketIn
+        // FIXME: We don't set the reassembly flags.
+        OFSetConfig sc = factory
+                .buildSetConfig()
+                .setMissSendLen((short) 0xffff)
+                .setXid(this.handshakeTransactionIds--)
+                .build();
+        msglist.add(sc);
+
+        // Barrier
+        OFBarrierRequest br = factory
+                .buildBarrierRequest()
+                .setXid(this.handshakeTransactionIds--)
+                .build();
+        msglist.add(br);
+
+        // Verify (need barrier?)
+        OFGetConfigRequest gcr = factory
+                .buildGetConfigRequest()
+                .setXid(this.handshakeTransactionIds--)
+                .build();
+        msglist.add(gcr);
+        channel.write(msglist);
+    }
+
+    /**
+     * send a description state request
+     * @throws IOException
+     */
+    private void sendHandshakeDescriptionStatsRequest() throws IOException {
+        // Get Description to set switch-specific flags
+        OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
+        OFDescStatsRequest dreq = factory
+                .buildDescStatsRequest()
+                .setXid(handshakeTransactionIds--)
+                .build();
+        channel.write(Collections.singletonList(dreq));
+    }
+
+    private void sendHandshakeOFPortDescRequest() throws IOException {
+        // Get port description for 1.3 switch
+        OFPortDescStatsRequest preq = factory13
+                .buildPortDescStatsRequest()
+                .setXid(handshakeTransactionIds--)
+                .build();
+        channel.write(Collections.singletonList(preq));
+    }
+
+    /**
+     * Read switch properties from storage and set switch attributes accordingly
+     */
+    private void readPropertyFromStorage() {
+        // XXX This is a placeholder for switch configuration
+    }
+
+    ChannelState getStateForTesting() {
+        return state;
+    }
+
+    void useRoleChangerWithOtherTimeoutForTesting(long roleTimeoutMs) {
+        roleChanger = new RoleChanger(roleTimeoutMs);
+    }
+
+
+}
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFChannelState.java b/src/main/java/net/floodlightcontroller/core/internal/OFChannelState.java
deleted file mode 100644
index 8130c0a..0000000
--- a/src/main/java/net/floodlightcontroller/core/internal/OFChannelState.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- *    Copyright 2011, Big Switch Networks, Inc.
- *    Originally created by David Erickson, Stanford University
- *
- *    Licensed under the Apache License, Version 2.0 (the "License"); you may
- *    not use this file except in compliance with the License. You may obtain
- *    a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- *    License for the specific language governing permissions and limitations
- *    under the License.
- **/
-
-package net.floodlightcontroller.core.internal;
-
-/**
- * Wrapper class to hold state for the OpenFlow switch connection
- *
- * @author readams
- */
-class OFChannelState {
-
-    /**
-     * State for handling the switch handshake
-     */
-    protected enum HandshakeState {
-        /**
-         * Beginning state
-         */
-        START,
-
-        /**
-         * Received HELLO from switch
-         */
-        HELLO,
-
-        /**
-         * We've received the features reply
-         * Waiting for Config and Description reply
-         */
-        FEATURES_REPLY,
-
-        /**
-         * Switch is ready for processing messages
-         */
-        READY
-
-    }
-
-    protected volatile HandshakeState hsState = HandshakeState.START;
-    protected boolean hasGetConfigReply = false;
-    protected boolean hasDescription = false;
-
-    // The firstRoleReplyRecevied flag indicates if we have received the
-    // first role reply message on this connection (in response to the 
-    // role request sent after the handshake). If role support is disabled
-    // on the controller we also set this flag to true. 
-    // The flag is used to decide if the flow table should be wiped
-    // @see Controller.handleRoleReplyMessage()
-    protected boolean firstRoleReplyReceived = false;
-}
\ No newline at end of file
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFFeaturesReplyFuture.java b/src/main/java/net/floodlightcontroller/core/internal/OFFeaturesReplyFuture.java
index 70ab58e..fa9f999 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/OFFeaturesReplyFuture.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/OFFeaturesReplyFuture.java
@@ -18,13 +18,13 @@
 
 import java.util.concurrent.TimeUnit;
 
+import org.projectfloodlight.openflow.protocol.OFFeaturesReply;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFType;
+
 import net.floodlightcontroller.core.IOFSwitch;
 import net.floodlightcontroller.threadpool.IThreadPoolService;
 
-import org.openflow.protocol.OFFeaturesReply;
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFType;
-
 /**
  * A concrete implementation that handles asynchronously receiving
  * OFFeaturesReply
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFMessageDecoder.java b/src/main/java/net/floodlightcontroller/core/internal/OFMessageDecoder.java
index 54fafd5..43257ca 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/OFMessageDecoder.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/OFMessageDecoder.java
@@ -23,39 +23,35 @@
 import org.jboss.netty.channel.Channel;
 import org.jboss.netty.channel.ChannelHandlerContext;
 import org.jboss.netty.handler.codec.frame.FrameDecoder;
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.factory.BasicFactory;
-import org.openflow.protocol.factory.OFMessageFactory;
+import org.projectfloodlight.openflow.protocol.OFFactories;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFMessageReader;
 
 /**
- * Decode an openflow message from a Channel, for use in a netty
- * pipeline
- *
- * @author readams
+ * Decode an openflow message from a Channel, for use in a netty pipeline
  */
 public class OFMessageDecoder extends FrameDecoder {
 
-    OFMessageFactory factory = new BasicFactory();
-
     @Override
     protected Object decode(ChannelHandlerContext ctx, Channel channel,
                             ChannelBuffer buffer) throws Exception {
         if (!channel.isConnected()) {
             // In testing, I see decode being called AFTER decode last.
-            // This check avoids that from reading curroupted frames
+            // This check avoids that from reading corrupted frames
             return null;
         }
 
-        List<OFMessage> message = factory.parseMessage(buffer);
-        return message;
-    }
+        // Note that a single call to decode results in reading a single
+        // OFMessage from the channel buffer, which is passed on to, and processed
+        // by, the controller (in OFChannelHandler).
+        // This is different from earlier behavior (with the original openflowj),
+        // where we parsed all the messages in the buffer, before passing on
+        // a list of the parsed messages to the controller.
+        // The performance *may or may not* not be as good as before.
+        OFMessageReader<OFMessage> reader = OFFactories.getGenericReader();
+		OFMessage message = reader.readFrom(buffer);
 
-    @Override
-    protected Object decodeLast(ChannelHandlerContext ctx, Channel channel,
-                                ChannelBuffer buffer) throws Exception {
-        // This is not strictly needed atthis time. It is used to detect
-        // connection reset detection from netty (for debug)
-        return null;
+		return message;
     }
 
 }
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFMessageEncoder.java b/src/main/java/net/floodlightcontroller/core/internal/OFMessageEncoder.java
index 29916c8..fdd6686 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/OFMessageEncoder.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/OFMessageEncoder.java
@@ -24,7 +24,8 @@
 import org.jboss.netty.channel.Channel;
 import org.jboss.netty.channel.ChannelHandlerContext;
 import org.jboss.netty.handler.codec.oneone.OneToOneEncoder;
-import org.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+
 
 /**
  * Encode an openflow message for output into a ChannelBuffer, for use in a
@@ -42,12 +43,13 @@
 
         @SuppressWarnings("unchecked")
         List<OFMessage> msglist = (List<OFMessage>) msg;
+        /* XXX S can't get length of OFMessage in loxigen's openflowj??
         int size = 0;
         for (OFMessage ofm : msglist) {
             size += ofm.getLengthU();
-        }
+        }*/
 
-        ChannelBuffer buf = ChannelBuffers.buffer(size);
+        ChannelBuffer buf = ChannelBuffers.dynamicBuffer();
 
         for (OFMessage ofm : msglist) {
             ofm.writeTo(buf);
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFMessageFuture.java b/src/main/java/net/floodlightcontroller/core/internal/OFMessageFuture.java
index 9e2a7ed..0d8d29f 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/OFMessageFuture.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/OFMessageFuture.java
@@ -23,11 +23,13 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFType;
+
 import net.floodlightcontroller.core.IOFSwitch;
 import net.floodlightcontroller.threadpool.IThreadPoolService;
 
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFType;
 
 /**
  * A Future object used to retrieve asynchronous OFMessage replies. Unregisters
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFStatisticsFuture.java b/src/main/java/net/floodlightcontroller/core/internal/OFStatisticsFuture.java
index 4651c74..91b81c1 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/OFStatisticsFuture.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/OFStatisticsFuture.java
@@ -24,10 +24,10 @@
 import net.floodlightcontroller.core.IOFSwitch;
 import net.floodlightcontroller.threadpool.IThreadPoolService;
 
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFStatisticsReply;
-import org.openflow.protocol.OFType;
-import org.openflow.protocol.statistics.OFStatistics;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFStatsReply;
+import org.projectfloodlight.openflow.protocol.OFStatsReplyFlags;
+import org.projectfloodlight.openflow.protocol.OFType;
 
 /**
  * A concrete implementation that handles asynchronously receiving OFStatistics
@@ -35,7 +35,7 @@
  * @author David Erickson (daviderickson@cs.stanford.edu)
  */
 public class OFStatisticsFuture extends
-        OFMessageFuture<List<OFStatistics>> {
+        OFMessageFuture<List<OFStatsReply>> {
 
     protected volatile boolean finished;
 
@@ -46,22 +46,23 @@
     }
 
     public OFStatisticsFuture(IThreadPoolService tp,
-                              IOFSwitch sw, int transactionId, long timeout, TimeUnit unit) {
+                              IOFSwitch sw, int transactionId, long timeout,
+                              TimeUnit unit) {
         super(tp, sw, OFType.STATS_REPLY, transactionId, timeout, unit);
         init();
     }
 
     private void init() {
         this.finished = false;
-        this.result = new CopyOnWriteArrayList<OFStatistics>();
+        this.result = new CopyOnWriteArrayList<OFStatsReply>();
     }
 
     @Override
     protected void handleReply(IOFSwitch sw, OFMessage msg) {
-        OFStatisticsReply sr = (OFStatisticsReply) msg;
+        OFStatsReply sr = (OFStatsReply) msg;
         synchronized (this.result) {
-            this.result.addAll(sr.getStatistics());
-            if ((sr.getFlags() & 0x1) == 0) {
+            this.result.add(sr);
+            if ( !(sr.getFlags().contains(OFStatsReplyFlags.REPLY_MORE)) ) {
                 this.finished = true;
             }
         }
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFSwitchImpl.java b/src/main/java/net/floodlightcontroller/core/internal/OFSwitchImpl.java
deleted file mode 100644
index 0bb9f2f..0000000
--- a/src/main/java/net/floodlightcontroller/core/internal/OFSwitchImpl.java
+++ /dev/null
@@ -1,877 +0,0 @@
-/**
- *    Copyright 2012, Big Switch Networks, Inc.
- *    Originally created by David Erickson, Stanford University
- *
- *    Licensed under the Apache License, Version 2.0 (the "License"); you may
- *    not use this file except in compliance with the License. You may obtain
- *    a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- *    License for the specific language governing permissions and limitations
- *    under the License.
- **/
-
-package net.floodlightcontroller.core.internal;
-
-import java.io.IOException;
-import java.net.SocketAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.WeakHashMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import net.floodlightcontroller.core.FloodlightContext;
-import net.floodlightcontroller.core.IFloodlightProviderService;
-import net.floodlightcontroller.core.IFloodlightProviderService.Role;
-import net.floodlightcontroller.core.IOFMessageListener;
-import net.floodlightcontroller.core.IOFSwitch;
-import net.floodlightcontroller.core.annotations.LogMessageDoc;
-import net.floodlightcontroller.core.annotations.LogMessageDocs;
-import net.floodlightcontroller.core.web.serializers.DPIDSerializer;
-import net.floodlightcontroller.threadpool.IThreadPoolService;
-import net.floodlightcontroller.util.TimedCache;
-
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.codehaus.jackson.map.ser.std.ToStringSerializer;
-import org.jboss.netty.channel.Channel;
-import org.openflow.protocol.OFFeaturesReply;
-import org.openflow.protocol.OFFeaturesRequest;
-import org.openflow.protocol.OFFlowMod;
-import org.openflow.protocol.OFMatch;
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFPhysicalPort;
-import org.openflow.protocol.OFPhysicalPort.OFPortConfig;
-import org.openflow.protocol.OFPhysicalPort.OFPortState;
-import org.openflow.protocol.OFPort;
-import org.openflow.protocol.OFStatisticsRequest;
-import org.openflow.protocol.OFType;
-import org.openflow.protocol.OFVendor;
-import org.openflow.protocol.statistics.OFDescriptionStatistics;
-import org.openflow.protocol.statistics.OFStatistics;
-import org.openflow.util.HexString;
-import org.openflow.util.U16;
-import org.openflow.vendor.nicira.OFNiciraVendorData;
-import org.openflow.vendor.nicira.OFRoleRequestVendorData;
-import org.openflow.vendor.nicira.OFRoleVendorData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * This is the internal representation of an openflow switch.
- */
-public class OFSwitchImpl implements IOFSwitch {
-    // TODO: should we really do logging in the class or should we throw
-    // exception that can then be handled by callers?
-    protected final static Logger log = LoggerFactory.getLogger(OFSwitchImpl.class);
-
-    private static final String HA_CHECK_SWITCH =
-            "Check the health of the indicated switch.  If the problem " +
-                    "persists or occurs repeatedly, it likely indicates a defect " +
-                    "in the switch HA implementation.";
-
-    protected ConcurrentMap<Object, Object> attributes;
-    protected IFloodlightProviderService floodlightProvider;
-    protected IThreadPoolService threadPool;
-    protected Date connectedSince;
-    protected String stringId;
-    protected Channel channel;
-    protected AtomicInteger transactionIdSource;
-    // Lock to protect modification of the port maps. We only need to
-    // synchronize on modifications. For read operations we are fine since
-    // we rely on ConcurrentMaps which works for our use case.
-    private Object portLock;
-    // Map port numbers to the appropriate OFPhysicalPort
-    protected ConcurrentHashMap<Short, OFPhysicalPort> portsByNumber;
-    // Map port names to the appropriate OFPhyiscalPort
-    // XXX: The OF spec doesn't specify if port names need to be unique but
-    //      according it's always the case in practice.
-    protected ConcurrentHashMap<String, OFPhysicalPort> portsByName;
-    protected Map<Integer, OFStatisticsFuture> statsFutureMap;
-    protected Map<Integer, IOFMessageListener> iofMsgListenersMap;
-    protected Map<Integer, OFFeaturesReplyFuture> featuresFutureMap;
-    protected boolean connected;
-    protected Role role;
-    protected TimedCache<Long> timedCache;
-    protected ReentrantReadWriteLock listenerLock;
-    protected ConcurrentMap<Short, Long> portBroadcastCacheHitMap;
-    /**
-     * When sending a role request message, the role request is added
-     * to this queue. If a role reply is received this queue is checked to
-     * verify that the reply matches the expected reply. We require in order
-     * delivery of replies. That's why we use a Queue.
-     * The RoleChanger uses a timeout to ensure we receive a timely reply.
-     * <p/>
-     * Need to synchronize on this instance if a request is sent, received,
-     * checked.
-     */
-    protected LinkedList<PendingRoleRequestEntry> pendingRoleRequests;
-
-    /* Switch features from initial featuresReply */
-    protected int capabilities;
-    protected int buffers;
-    protected int actions;
-    protected byte tables;
-    protected long datapathId;
-
-    public static IOFSwitchFeatures switchFeatures;
-    protected static final ThreadLocal<Map<OFSwitchImpl, List<OFMessage>>> local_msg_buffer =
-            new ThreadLocal<Map<OFSwitchImpl, List<OFMessage>>>() {
-                @Override
-                protected Map<OFSwitchImpl, List<OFMessage>> initialValue() {
-                    return new WeakHashMap<OFSwitchImpl, List<OFMessage>>();
-                }
-            };
-
-    // for managing our map sizes
-    protected static final int MAX_MACS_PER_SWITCH = 1000;
-
-    protected static class PendingRoleRequestEntry {
-        protected int xid;
-        protected Role role;
-        // cookie is used to identify the role "generation". roleChanger uses
-        protected long cookie;
-
-        public PendingRoleRequestEntry(int xid, Role role, long cookie) {
-            this.xid = xid;
-            this.role = role;
-            this.cookie = cookie;
-        }
-    }
-
-    public OFSwitchImpl() {
-        this.stringId = null;
-        this.attributes = new ConcurrentHashMap<Object, Object>();
-        this.connectedSince = new Date();
-        this.transactionIdSource = new AtomicInteger();
-        this.portLock = new Object();
-        this.portsByNumber = new ConcurrentHashMap<Short, OFPhysicalPort>();
-        this.portsByName = new ConcurrentHashMap<String, OFPhysicalPort>();
-        this.connected = true;
-        this.statsFutureMap = new ConcurrentHashMap<Integer, OFStatisticsFuture>();
-        this.featuresFutureMap = new ConcurrentHashMap<Integer, OFFeaturesReplyFuture>();
-        this.iofMsgListenersMap = new ConcurrentHashMap<Integer, IOFMessageListener>();
-        this.role = null;
-        this.timedCache = new TimedCache<Long>(100, 5 * 1000);  // 5 seconds interval
-        this.listenerLock = new ReentrantReadWriteLock();
-        this.portBroadcastCacheHitMap = new ConcurrentHashMap<Short, Long>();
-        this.pendingRoleRequests = new LinkedList<OFSwitchImpl.PendingRoleRequestEntry>();
-
-        // Defaults properties for an ideal switch
-        this.setAttribute(PROP_FASTWILDCARDS, OFMatch.OFPFW_ALL);
-        this.setAttribute(PROP_SUPPORTS_OFPP_FLOOD, new Boolean(true));
-        this.setAttribute(PROP_SUPPORTS_OFPP_TABLE, new Boolean(true));
-    }
-
-
-    @Override
-    public Object getAttribute(String name) {
-        if (this.attributes.containsKey(name)) {
-            return this.attributes.get(name);
-        }
-        return null;
-    }
-
-    @Override
-    public void setAttribute(String name, Object value) {
-        this.attributes.put(name, value);
-        return;
-    }
-
-    @Override
-    public Object removeAttribute(String name) {
-        return this.attributes.remove(name);
-    }
-
-    @Override
-    public boolean hasAttribute(String name) {
-        return this.attributes.containsKey(name);
-    }
-
-    @Override
-    @JsonIgnore
-    public Channel getChannel() {
-        return this.channel;
-    }
-
-    @JsonIgnore
-    public void setChannel(Channel channel) {
-        this.channel = channel;
-    }
-
-    @Override
-    public void write(OFMessage m, FloodlightContext bc) throws IOException {
-        Map<OFSwitchImpl, List<OFMessage>> msg_buffer_map = local_msg_buffer.get();
-        List<OFMessage> msg_buffer = msg_buffer_map.get(this);
-        if (msg_buffer == null) {
-            msg_buffer = new ArrayList<OFMessage>();
-            msg_buffer_map.put(this, msg_buffer);
-        }
-
-        this.floodlightProvider.handleOutgoingMessage(this, m, bc);
-        msg_buffer.add(m);
-
-        if ((msg_buffer.size() >= Controller.BATCH_MAX_SIZE) ||
-                ((m.getType() != OFType.PACKET_OUT) && (m.getType() != OFType.FLOW_MOD))) {
-            this.write(msg_buffer);
-            msg_buffer.clear();
-        }
-    }
-
-    @Override
-    @LogMessageDoc(level = "WARN",
-            message = "Sending OF message that modifies switch " +
-                    "state while in the slave role: {switch}",
-            explanation = "An application has sent a message to a switch " +
-                    "that is not valid when the switch is in a slave role",
-            recommendation = LogMessageDoc.REPORT_CONTROLLER_BUG)
-    public void write(List<OFMessage> msglist,
-                      FloodlightContext bc) throws IOException {
-        for (OFMessage m : msglist) {
-            if (role == Role.SLAVE) {
-                switch (m.getType()) {
-                    case PACKET_OUT:
-                    case FLOW_MOD:
-                    case PORT_MOD:
-                        log.warn("Sending OF message that modifies switch " +
-                                "state while in the slave role: {}",
-                                m.getType().name());
-                        break;
-                    default:
-                        break;
-                }
-            }
-            this.floodlightProvider.handleOutgoingMessage(this, m, bc);
-        }
-        this.write(msglist);
-    }
-
-    public void write(List<OFMessage> msglist) throws IOException {
-        this.channel.write(msglist);
-    }
-
-    @Override
-    public void disconnectOutputStream() {
-        channel.close();
-    }
-
-    @Override
-    @JsonIgnore
-    public void setFeaturesReply(OFFeaturesReply featuresReply) {
-        synchronized (portLock) {
-            if (stringId == null) {
-                /* ports are updated via port status message, so we
-                 * only fill in ports on initial connection.
-                 */
-                for (OFPhysicalPort port : featuresReply.getPorts()) {
-                    setPort(port);
-                }
-            }
-            this.datapathId = featuresReply.getDatapathId();
-            this.capabilities = featuresReply.getCapabilities();
-            this.buffers = featuresReply.getBuffers();
-            this.actions = featuresReply.getActions();
-            this.tables = featuresReply.getTables();
-            this.stringId = HexString.toHexString(this.datapathId);
-        }
-    }
-
-    @Override
-    @JsonIgnore
-    public Collection<OFPhysicalPort> getEnabledPorts() {
-        List<OFPhysicalPort> result = new ArrayList<OFPhysicalPort>();
-        for (OFPhysicalPort port : portsByNumber.values()) {
-            if (portEnabled(port)) {
-                result.add(port);
-            }
-        }
-        return result;
-    }
-
-    @Override
-    @JsonIgnore
-    public Collection<Short> getEnabledPortNumbers() {
-        List<Short> result = new ArrayList<Short>();
-        for (OFPhysicalPort port : portsByNumber.values()) {
-            if (portEnabled(port)) {
-                result.add(port.getPortNumber());
-            }
-        }
-        return result;
-    }
-
-    @Override
-    public OFPhysicalPort getPort(short portNumber) {
-        return portsByNumber.get(portNumber);
-    }
-
-    @Override
-    public OFPhysicalPort getPort(String portName) {
-        return portsByName.get(portName);
-    }
-
-    @Override
-    @JsonIgnore
-    public void setPort(OFPhysicalPort port) {
-        synchronized (portLock) {
-            portsByNumber.put(port.getPortNumber(), port);
-            portsByName.put(port.getName(), port);
-        }
-    }
-
-    @Override
-    @JsonProperty("ports")
-    public Collection<OFPhysicalPort> getPorts() {
-        return Collections.unmodifiableCollection(portsByNumber.values());
-    }
-
-    @Override
-    public void deletePort(short portNumber) {
-        synchronized (portLock) {
-            portsByName.remove(portsByNumber.get(portNumber).getName());
-            portsByNumber.remove(portNumber);
-        }
-    }
-
-    @Override
-    public void deletePort(String portName) {
-        synchronized (portLock) {
-            portsByNumber.remove(portsByName.get(portName).getPortNumber());
-            portsByName.remove(portName);
-        }
-    }
-
-    @Override
-    public boolean portEnabled(short portNumber) {
-        if (portsByNumber.get(portNumber) == null) return false;
-        return portEnabled(portsByNumber.get(portNumber));
-    }
-
-    @Override
-    public boolean portEnabled(String portName) {
-        if (portsByName.get(portName) == null) return false;
-        return portEnabled(portsByName.get(portName));
-    }
-
-    @Override
-    public boolean portEnabled(OFPhysicalPort port) {
-        if (port == null)
-            return false;
-        if ((port.getConfig() & OFPortConfig.OFPPC_PORT_DOWN.getValue()) > 0)
-            return false;
-        if ((port.getState() & OFPortState.OFPPS_LINK_DOWN.getValue()) > 0)
-            return false;
-        // Port STP state doesn't work with multiple VLANs, so ignore it for now
-        //if ((port.getState() & OFPortState.OFPPS_STP_MASK.getValue()) == OFPortState.OFPPS_STP_BLOCK.getValue())
-        //    return false;
-        return true;
-    }
-
-    @Override
-    @JsonSerialize(using = DPIDSerializer.class)
-    @JsonProperty("dpid")
-    public long getId() {
-        if (this.stringId == null)
-            throw new RuntimeException("Features reply has not yet been set");
-        return this.datapathId;
-    }
-
-    @JsonIgnore
-    @Override
-    public String getStringId() {
-        return stringId;
-    }
-
-    /* (non-Javadoc)
-     * @see java.lang.Object#toString()
-     */
-    @Override
-    public String toString() {
-        return "OFSwitchImpl [" + ((channel != null) ? channel.getRemoteAddress() : "?") + " DPID[" + ((stringId != null) ? stringId : "?") + "]]";
-    }
-
-    @Override
-    public ConcurrentMap<Object, Object> getAttributes() {
-        return this.attributes;
-    }
-
-    @Override
-    public Date getConnectedSince() {
-        return connectedSince;
-    }
-
-    @JsonIgnore
-    @Override
-    public int getNextTransactionId() {
-        return this.transactionIdSource.incrementAndGet();
-    }
-
-    @Override
-    public void sendStatsQuery(OFStatisticsRequest request, int xid,
-                               IOFMessageListener caller) throws IOException {
-        request.setXid(xid);
-        this.iofMsgListenersMap.put(xid, caller);
-        List<OFMessage> msglist = new ArrayList<OFMessage>(1);
-        msglist.add(request);
-        this.channel.write(msglist);
-        return;
-    }
-
-    @Override
-    public Future<List<OFStatistics>> getStatistics(OFStatisticsRequest request) throws IOException {
-        request.setXid(getNextTransactionId());
-        OFStatisticsFuture future = new OFStatisticsFuture(threadPool, this, request.getXid());
-        this.statsFutureMap.put(request.getXid(), future);
-        List<OFMessage> msglist = new ArrayList<OFMessage>(1);
-        msglist.add(request);
-        this.channel.write(msglist);
-        return future;
-    }
-
-    @Override
-    public void deliverStatisticsReply(OFMessage reply) {
-        OFStatisticsFuture future = this.statsFutureMap.get(reply.getXid());
-        if (future != null) {
-            future.deliverFuture(this, reply);
-            // The future will ultimately unregister itself and call
-            // cancelStatisticsReply
-            return;
-        }
-        /* Transaction id was not found in statsFutureMap.check the other map */
-        IOFMessageListener caller = this.iofMsgListenersMap.get(reply.getXid());
-        if (caller != null) {
-            caller.receive(this, reply, null);
-        }
-    }
-
-    @Override
-    public void cancelStatisticsReply(int transactionId) {
-        if (null == this.statsFutureMap.remove(transactionId)) {
-            this.iofMsgListenersMap.remove(transactionId);
-        }
-    }
-
-    @Override
-    public void cancelAllStatisticsReplies() {
-        /* we don't need to be synchronized here. Even if another thread
-         * modifies the map while we're cleaning up the future will eventuall
-         * timeout */
-        for (OFStatisticsFuture f : statsFutureMap.values()) {
-            f.cancel(true);
-        }
-        statsFutureMap.clear();
-        iofMsgListenersMap.clear();
-    }
-
-
-    /**
-     * @param floodlightProvider the floodlightProvider to set
-     */
-    @JsonIgnore
-    public void setFloodlightProvider(IFloodlightProviderService floodlightProvider) {
-        this.floodlightProvider = floodlightProvider;
-    }
-
-    @JsonIgnore
-    public void setThreadPoolService(IThreadPoolService tp) {
-        this.threadPool = tp;
-    }
-
-    @JsonIgnore
-    @Override
-    public synchronized boolean isConnected() {
-        return connected;
-    }
-
-    @Override
-    @JsonIgnore
-    public synchronized void setConnected(boolean connected) {
-        this.connected = connected;
-    }
-
-    @Override
-    public Role getRole() {
-        return role;
-    }
-
-    @JsonIgnore
-    @Override
-    public boolean isActive() {
-        return (role != Role.SLAVE);
-    }
-
-    @Override
-    @JsonIgnore
-    public void setSwitchProperties(OFDescriptionStatistics description) {
-        if (switchFeatures != null) {
-            switchFeatures.setFromDescription(this, description);
-        }
-    }
-
-    @Override
-    @LogMessageDoc(level = "ERROR",
-            message = "Failed to clear all flows on switch {switch}",
-            explanation = "An I/O error occured while trying to clear " +
-                    "flows on the switch.",
-            recommendation = LogMessageDoc.CHECK_SWITCH)
-    public void clearAllFlowMods() {
-        // Delete all pre-existing flows
-        OFMatch match = new OFMatch().setWildcards(OFMatch.OFPFW_ALL);
-        OFMessage fm = ((OFFlowMod) floodlightProvider.getOFMessageFactory()
-                .getMessage(OFType.FLOW_MOD))
-                .setMatch(match)
-                .setCommand(OFFlowMod.OFPFC_DELETE)
-                .setOutPort(OFPort.OFPP_NONE)
-                .setLength(U16.t(OFFlowMod.MINIMUM_LENGTH));
-        try {
-            List<OFMessage> msglist = new ArrayList<OFMessage>(1);
-            msglist.add(fm);
-            channel.write(msglist);
-        } catch (Exception e) {
-            log.error("Failed to clear all flows on switch " + this, e);
-        }
-    }
-
-    @Override
-    public boolean updateBroadcastCache(Long entry, Short port) {
-        if (timedCache.update(entry)) {
-            Long count = portBroadcastCacheHitMap.putIfAbsent(port, new Long(1));
-            if (count != null) {
-                count++;
-            }
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    @Override
-    @JsonIgnore
-    public Map<Short, Long> getPortBroadcastHits() {
-        return this.portBroadcastCacheHitMap;
-    }
-
-
-    @Override
-    public void flush() {
-        Map<OFSwitchImpl, List<OFMessage>> msg_buffer_map = local_msg_buffer.get();
-        List<OFMessage> msglist = msg_buffer_map.get(this);
-        if ((msglist != null) && (msglist.size() > 0)) {
-            try {
-                this.write(msglist);
-            } catch (IOException e) {
-                log.error("Failed flushing messages", e);
-            }
-            msglist.clear();
-        }
-    }
-
-    public static void flush_all() {
-        Map<OFSwitchImpl, List<OFMessage>> msg_buffer_map = local_msg_buffer.get();
-        for (OFSwitchImpl sw : msg_buffer_map.keySet()) {
-            sw.flush();
-        }
-    }
-
-    /**
-     * Return a read lock that must be held while calling the listeners for
-     * messages from the switch. Holding the read lock prevents the active
-     * switch list from being modified out from under the listeners.
-     *
-     * @return
-     */
-    @JsonIgnore
-    public Lock getListenerReadLock() {
-        return listenerLock.readLock();
-    }
-
-    /**
-     * Return a write lock that must be held when the controllers modifies the
-     * list of active switches. This is to ensure that the active switch list
-     * doesn't change out from under the listeners as they are handling a
-     * message from the switch.
-     *
-     * @return
-     */
-    @JsonIgnore
-    public Lock getListenerWriteLock() {
-        return listenerLock.writeLock();
-    }
-
-    /**
-     * Get the IP Address for the switch
-     *
-     * @return the inet address
-     */
-    @JsonSerialize(using = ToStringSerializer.class)
-    public SocketAddress getInetAddress() {
-        return channel.getRemoteAddress();
-    }
-
-    /**
-     * Send NX role request message to the switch requesting the specified role.
-     * <p/>
-     * This method should ONLY be called by @see RoleChanger.submitRequest().
-     * <p/>
-     * After sending the request add it to the queue of pending request. We
-     * use the queue to later verify that we indeed receive the correct reply.
-     *
-     * @param sw     switch to send the role request message to
-     * @param role   role to request
-     * @param cookie an opaque value that will be stored in the pending queue so
-     *               RoleChanger can check for timeouts.
-     * @return transaction id of the role request message that was sent
-     */
-    protected int sendNxRoleRequest(Role role, long cookie)
-            throws IOException {
-        synchronized (pendingRoleRequests) {
-            // Convert the role enum to the appropriate integer constant used
-            // in the NX role request message
-            int nxRole = 0;
-            switch (role) {
-                case EQUAL:
-                    nxRole = OFRoleVendorData.NX_ROLE_OTHER;
-                    break;
-                case MASTER:
-                    nxRole = OFRoleVendorData.NX_ROLE_MASTER;
-                    break;
-                case SLAVE:
-                    nxRole = OFRoleVendorData.NX_ROLE_SLAVE;
-                    break;
-                default:
-                    log.error("Invalid Role specified for switch {}."
-                            + " Disconnecting.", this);
-                    // TODO: should throw an error
-                    return 0;
-            }
-
-            // Construct the role request message
-            OFVendor roleRequest = (OFVendor) floodlightProvider.
-                    getOFMessageFactory().getMessage(OFType.VENDOR);
-            int xid = this.getNextTransactionId();
-            roleRequest.setXid(xid);
-            roleRequest.setVendor(OFNiciraVendorData.NX_VENDOR_ID);
-            OFRoleRequestVendorData roleRequestData = new OFRoleRequestVendorData();
-            roleRequestData.setRole(nxRole);
-            roleRequest.setVendorData(roleRequestData);
-            roleRequest.setLengthU(OFVendor.MINIMUM_LENGTH +
-                    roleRequestData.getLength());
-
-            // Send it to the switch
-            List<OFMessage> msglist = new ArrayList<OFMessage>(1);
-            msglist.add(roleRequest);
-            // FIXME: should this use this.write() in order for messages to
-            // be processed by handleOutgoingMessage()
-            this.channel.write(msglist);
-
-            pendingRoleRequests.add(new PendingRoleRequestEntry(xid, role, cookie));
-            return xid;
-        }
-    }
-
-    /**
-     * Deliver a RoleReply message to this switch. Checks if the reply
-     * message matches the expected reply (head of the pending request queue).
-     * We require in-order delivery of replies. If there's any deviation from
-     * our expectations we disconnect the switch.
-     * <p/>
-     * We must not check the received role against the controller's current
-     * role because there's no synchronization but that's fine @see RoleChanger
-     * <p/>
-     * Will be called by the OFChannelHandler's receive loop
-     *
-     * @param xid  Xid of the reply message
-     * @param role The Role in the the reply message
-     */
-    @LogMessageDocs({
-            @LogMessageDoc(level = "ERROR",
-                    message = "Switch {switch}: received unexpected role reply for " +
-                            "Role {role}" +
-                            " Disconnecting switch",
-                    explanation = "The switch sent an unexpected HA role reply",
-                    recommendation = HA_CHECK_SWITCH),
-            @LogMessageDoc(level = "ERROR",
-                    message = "Switch {switch}: expected role reply with " +
-                            "Xid {xid}, got {xid}. Disconnecting switch",
-                    explanation = "The switch sent an unexpected HA role reply",
-                    recommendation = HA_CHECK_SWITCH),
-            @LogMessageDoc(level = "ERROR",
-                    message = "Switch {switch}: expected role reply with " +
-                            "Role {role}, got {role}. Disconnecting switch",
-                    explanation = "The switch sent an unexpected HA role reply",
-                    recommendation = HA_CHECK_SWITCH)
-    })
-    protected void deliverRoleReply(int xid, Role role) {
-        synchronized (pendingRoleRequests) {
-            PendingRoleRequestEntry head = pendingRoleRequests.poll();
-            if (head == null) {
-                // Maybe don't disconnect if the role reply we received is
-                // for the same role we are already in.
-                log.error("Switch {}: received unexpected role reply for Role {}" +
-                        " Disconnecting switch", this, role);
-                this.channel.close();
-            } else if (head.xid != xid) {
-                // check xid before role!!
-                log.error("Switch {}: expected role reply with " +
-                        "Xid {}, got {}. Disconnecting switch",
-                        new Object[]{this, head.xid, xid});
-                this.channel.close();
-            } else if (head.role != role) {
-                log.error("Switch {}: expected role reply with " +
-                        "Role {}, got {}. Disconnecting switch",
-                        new Object[]{this, head.role, role});
-                this.channel.close();
-            } else {
-                log.debug("Received role reply message from {}, setting role to {}",
-                        this, role);
-                if (this.role == null && getAttribute(SWITCH_SUPPORTS_NX_ROLE) == null) {
-                    // The first role reply we received. Set the attribute
-                    // that the switch supports roles
-                    setAttribute(SWITCH_SUPPORTS_NX_ROLE, true);
-                }
-                this.role = role;
-            }
-        }
-    }
-
-    /**
-     * Checks whether the given xid matches the xid of the first pending
-     * role request.
-     *
-     * @param xid
-     * @return
-     */
-    protected boolean checkFirstPendingRoleRequestXid(int xid) {
-        synchronized (pendingRoleRequests) {
-            PendingRoleRequestEntry head = pendingRoleRequests.peek();
-            if (head == null)
-                return false;
-            else
-                return head.xid == xid;
-        }
-    }
-
-    /**
-     * Checks whether the given request cookie matches the cookie of the first
-     * pending request
-     *
-     * @param cookie
-     * @return
-     */
-    protected boolean checkFirstPendingRoleRequestCookie(long cookie) {
-        synchronized (pendingRoleRequests) {
-            PendingRoleRequestEntry head = pendingRoleRequests.peek();
-            if (head == null)
-                return false;
-            else
-                return head.cookie == cookie;
-        }
-    }
-
-    /**
-     * Called if we receive a vendor error message indicating that roles
-     * are not supported by the switch. If the xid matches the first pending
-     * one, we'll mark the switch as not supporting roles and remove the head.
-     * Otherwise we ignore it.
-     *
-     * @param xid
-     */
-    protected void deliverRoleRequestNotSupported(int xid) {
-        deliverRoleRequestNotSupportedEx(xid);
-    }
-
-    /**
-     * ONOS Extension to deliverRoleRequestNotSupported().
-     * This version return the Roll request made.
-     *
-     * @param xid
-     * @return Role of attempted RoleRequest.
-     * @see deliverRoleRequestNotSupported
-     */
-    protected Role deliverRoleRequestNotSupportedEx(int xid) {
-        synchronized (pendingRoleRequests) {
-            PendingRoleRequestEntry head = pendingRoleRequests.poll();
-            this.role = null;
-            if (head != null && head.xid == xid) {
-                setAttribute(SWITCH_SUPPORTS_NX_ROLE, false);
-                return head.role;
-            } else {
-                log.debug("Closing {} because a role request error didn't match " +
-                        "head of pendingRoleRequests queue", this);
-                this.channel.close();
-                return null;
-            }
-        }
-    }
-
-    @Override
-    public Future<OFFeaturesReply> getFeaturesReplyFromSwitch()
-            throws IOException {
-        OFMessage request = new OFFeaturesRequest();
-        request.setXid(getNextTransactionId());
-        OFFeaturesReplyFuture future =
-                new OFFeaturesReplyFuture(threadPool, this, request.getXid());
-        this.featuresFutureMap.put(request.getXid(), future);
-        List<OFMessage> msglist = new ArrayList<OFMessage>(1);
-        msglist.add(request);
-        this.channel.write(msglist);
-        return future;
-    }
-
-    @Override
-    public void deliverOFFeaturesReply(OFMessage reply) {
-        OFFeaturesReplyFuture future = this.featuresFutureMap.get(reply.getXid());
-        if (future != null) {
-            future.deliverFuture(this, reply);
-            // The future will ultimately unregister itself and call
-            // cancelFeaturesReply
-            return;
-        }
-        log.error("Switch {}: received unexpected featureReply", this);
-    }
-
-    @Override
-    public void cancelFeaturesReply(int transactionId) {
-        this.featuresFutureMap.remove(transactionId);
-    }
-
-
-    @Override
-    public int getBuffers() {
-        return buffers;
-    }
-
-
-    @Override
-    public int getActions() {
-        return actions;
-    }
-
-
-    @Override
-    public int getCapabilities() {
-        return capabilities;
-    }
-
-
-    @Override
-    public byte getTables() {
-        return tables;
-    }
-}
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFSwitchImplBase.java b/src/main/java/net/floodlightcontroller/core/internal/OFSwitchImplBase.java
new file mode 100644
index 0000000..c3d6e63
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/core/internal/OFSwitchImplBase.java
@@ -0,0 +1,1288 @@
+package net.floodlightcontroller.core.internal;
+
+/**
+ *    Copyright 2012, Big Switch Networks, Inc.
+ *    Originally created by David Erickson, Stanford University
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import net.floodlightcontroller.core.FloodlightContext;
+import net.floodlightcontroller.core.IFloodlightProviderService;
+import net.floodlightcontroller.core.IFloodlightProviderService.Role;
+import net.floodlightcontroller.core.IOFMessageListener;
+import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeAlreadyStarted;
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeCompleted;
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeNotStarted;
+import net.floodlightcontroller.core.annotations.LogMessageDoc;
+import net.floodlightcontroller.core.web.serializers.DPIDSerializer;
+import net.floodlightcontroller.debugcounter.IDebugCounter;
+import net.floodlightcontroller.debugcounter.IDebugCounterService;
+import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterException;
+import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType;
+import net.floodlightcontroller.debugcounter.NullDebugCounter;
+import net.floodlightcontroller.threadpool.IThreadPoolService;
+import net.floodlightcontroller.util.LinkedHashSetWrapper;
+import net.floodlightcontroller.util.OrderedCollection;
+import net.floodlightcontroller.util.TimedCache;
+
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.jboss.netty.channel.Channel;
+import org.projectfloodlight.openflow.protocol.OFActionType;
+import org.projectfloodlight.openflow.protocol.OFCapabilities;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFFeaturesReply;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFPortConfig;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFPortReason;
+import org.projectfloodlight.openflow.protocol.OFPortState;
+import org.projectfloodlight.openflow.protocol.OFPortStatus;
+import org.projectfloodlight.openflow.protocol.OFStatsReply;
+import org.projectfloodlight.openflow.protocol.OFStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.types.DatapathId;
+import org.projectfloodlight.openflow.types.OFAuxId;
+import org.projectfloodlight.openflow.types.OFGroup;
+import org.projectfloodlight.openflow.types.OFPort;
+import org.projectfloodlight.openflow.types.TableId;
+import org.projectfloodlight.openflow.types.U64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * This is the internal representation of an openflow switch.
+ */
+public class OFSwitchImplBase implements IOFSwitch {
+    // TODO: should we really do logging in the class or should we throw
+    // exception that can then be handled by callers?
+    protected final static Logger log = LoggerFactory.getLogger(OFSwitchImplBase.class);
+
+    private static final String HA_CHECK_SWITCH =
+            "Check the health of the indicated switch.  If the problem " +
+                    "persists or occurs repeatedly, it likely indicates a defect " +
+                    "in the switch HA implementation.";
+
+    protected ConcurrentMap<Object, Object> attributes;
+    protected IFloodlightProviderService floodlightProvider;
+    protected IThreadPoolService threadPool;
+    protected Date connectedSince;
+    protected String stringId;
+    protected Channel channel;
+    // transaction id used for messages sent out to this switch from
+    // this controller instance. This xid has significance only between this
+    // controller<->switch pair.
+    protected AtomicInteger transactionIdSource;
+
+    // generation id used for roleRequest messages sent to switches (see section
+    // 6.3.5 of the OF1.3.4 spec). This generationId has significance between
+    // all the controllers that this switch is connected to; and only for role
+    // request messages with role MASTER or SLAVE. The set of Controllers that
+    // this switch is connected to should coordinate the next generation id,
+    // via transactional semantics.
+    protected long generationIdSource;
+
+    // Lock to protect modification of the port maps. We only need to
+    // synchronize on modifications. For read operations we are fine since
+    // we rely on ConcurrentMaps which works for our use case.
+    //private Object portLock; XXX S remove this
+
+    // Map port numbers to the appropriate OFPortDesc
+    protected ConcurrentHashMap<Integer, OFPortDesc> portsByNumber;
+    // Map port names to the appropriate OFPhyiscalPort
+    // XXX: The OF spec doesn't specify if port names need to be unique but
+    //      according it's always the case in practice.
+    protected ConcurrentHashMap<String, OFPortDesc> portsByName;
+    protected Map<Integer, OFStatisticsFuture> statsFutureMap;
+    protected Map<Integer, IOFMessageListener> iofMsgListenersMap; // XXX S why is this needed?
+    protected Map<Integer, OFFeaturesReplyFuture> featuresFutureMap;
+    protected boolean connected;
+    protected Role role;
+    protected TimedCache<Long> timedCache;
+    protected ReentrantReadWriteLock listenerLock;
+    protected ConcurrentMap<Short, Long> portBroadcastCacheHitMap;
+    /**
+     * When sending a role request message, the role request is added
+     * to this queue. If a role reply is received this queue is checked to
+     * verify that the reply matches the expected reply. We require in order
+     * delivery of replies. That's why we use a Queue.
+     * The RoleChanger uses a timeout to ensure we receive a timely reply.
+     * <p/>
+     * Need to synchronize on this instance if a request is sent, received,
+     * checked.
+     */
+    protected LinkedList<PendingRoleRequestEntry> pendingRoleRequests;
+
+    /** OpenFlow version for this switch */
+    protected OFVersion ofversion;
+    // Description stats reply describing this switch
+    private OFDescStatsReply switchDescription;
+    // Switch features from initial featuresReply
+    protected Set<OFCapabilities> capabilities;
+    protected int buffers;
+    protected Set<OFActionType> actions;
+    protected byte tables;
+    protected DatapathId datapathId;
+	private OFAuxId auxId;
+
+    private IDebugCounterService debugCounters;
+	private boolean debugCountersRegistered;
+	@SuppressWarnings("unused")
+    private IDebugCounter ctrSwitch, ctrSwitchPktin, ctrSwitchWrite,
+    ctrSwitchPktinDrops, ctrSwitchWriteDrops;
+
+	protected boolean startDriverHandshakeCalled = false;
+	private boolean flowTableFull = false;
+
+	private final PortManager portManager;
+
+
+
+    protected static final ThreadLocal<Map<OFSwitchImplBase, List<OFMessage>>> local_msg_buffer =
+		new ThreadLocal<Map<OFSwitchImplBase, List<OFMessage>>>() {
+	@Override
+	protected Map<OFSwitchImplBase, List<OFMessage>> initialValue() {
+		return new WeakHashMap<OFSwitchImplBase, List<OFMessage>>();
+	}
+    };
+
+
+	private static final String BASE = "switchbase";
+
+    protected static class PendingRoleRequestEntry {
+        protected int xid;
+        protected Role role;
+        // cookie is used to identify the role "generation". roleChanger uses
+        protected long cookie;
+
+        public PendingRoleRequestEntry(int xid, Role role, long cookie) {
+            this.xid = xid;
+            this.role = role;
+            this.cookie = cookie;
+        }
+    }
+
+    public OFSwitchImplBase() {
+        this.stringId = null;
+        this.attributes = new ConcurrentHashMap<Object, Object>();
+        this.connectedSince = new Date();
+        this.transactionIdSource = new AtomicInteger();
+        this.generationIdSource = 0; // XXX S this is wrong; should be negotiated
+        // XXX S no need this.portLock = new Object();
+        this.portsByNumber = new ConcurrentHashMap<Integer, OFPortDesc>();
+        this.portsByName = new ConcurrentHashMap<String, OFPortDesc>();
+        this.connected = true;
+        this.statsFutureMap = new ConcurrentHashMap<Integer, OFStatisticsFuture>();
+        this.featuresFutureMap = new ConcurrentHashMap<Integer, OFFeaturesReplyFuture>();
+        this.iofMsgListenersMap = new ConcurrentHashMap<Integer, IOFMessageListener>();
+        this.role = null;
+        this.timedCache = new TimedCache<Long>(100, 5 * 1000);  // 5 seconds interval
+        this.listenerLock = new ReentrantReadWriteLock();
+        this.pendingRoleRequests = new LinkedList<OFSwitchImplBase.PendingRoleRequestEntry>();
+        this.portManager = new PortManager();
+        // by default the base impl declares no support for Nx_role_requests.
+        // OF1.0 switches like OVS that do support these messages should set the
+        // attribute in the associated switch driver.
+        setAttribute(SWITCH_SUPPORTS_NX_ROLE, false);
+
+    }
+
+    //*******************************************
+    //    Setters and Getters
+    //*******************************************
+
+    @Override
+    public Object getAttribute(String name) {
+        if (this.attributes.containsKey(name)) {
+            return this.attributes.get(name);
+        }
+        return null;
+    }
+
+    @Override
+    public ConcurrentMap<Object, Object> getAttributes() {
+        return this.attributes;
+    }
+
+    @Override
+    public void setAttribute(String name, Object value) {
+        this.attributes.put(name, value);
+        return;
+    }
+
+    @Override
+    public Object removeAttribute(String name) {
+        return this.attributes.remove(name);
+    }
+
+    @Override
+    public boolean hasAttribute(String name) {
+        return this.attributes.containsKey(name);
+    }
+
+    @Override
+    @JsonSerialize(using = DPIDSerializer.class)
+    @JsonProperty("dpid")
+    public long getId() {
+        if (this.stringId == null)
+            throw new RuntimeException("Features reply has not yet been set");
+        return this.datapathId.getLong();
+    }
+
+    @JsonIgnore
+    @Override
+    public String getStringId() {
+        return stringId;
+    }
+
+    /** Retrieve the openflow version (eg. OF1.0, OF1.3) for this switch
+     */
+    public OFVersion getOFVersion() {
+	return ofversion;
+    }
+
+    public void setOFVersion(OFVersion ofv) {
+	ofversion = ofv;
+    }
+
+    /**
+     * @param floodlightProvider the floodlightProvider to set
+     */
+    @JsonIgnore
+    @Override
+    public void setFloodlightProvider(IFloodlightProviderService floodlightProvider) {
+        this.floodlightProvider = floodlightProvider;
+    }
+
+    @JsonIgnore
+    @Override
+    public void setThreadPoolService(IThreadPoolService tp) {
+        this.threadPool = tp;
+    }
+
+    @Override
+    @JsonIgnore
+    public void setDebugCounterService(IDebugCounterService debugCounters)
+            throws CounterException {
+        this.debugCounters = debugCounters;
+        registerOverloadCounters();
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        return "OFSwitchImpl [" + ((channel != null) ? channel.getRemoteAddress() : "?")
+			+ " DPID[" + ((stringId != null) ? stringId : "?") + "]]";
+    }
+
+
+    //*******************************************
+    //    Channel related methods
+    //*******************************************
+
+    @JsonIgnore
+    @Override
+    public void setChannel(Channel channel) {
+        this.channel = channel;
+    }
+
+    @Override
+    public void write(OFMessage m, FloodlightContext bc) throws IOException {
+        Map<OFSwitchImplBase, List<OFMessage>> msg_buffer_map = local_msg_buffer.get();
+        List<OFMessage> msg_buffer = msg_buffer_map.get(this);
+        if (msg_buffer == null) {
+            msg_buffer = new ArrayList<OFMessage>();
+            msg_buffer_map.put(this, msg_buffer);
+        }
+        // XXX S will change when iFloodlight provider changes
+        //this.floodlightProvider.handleOutgoingMessage(this, m, bc);
+        msg_buffer.add(m);
+
+        if ((msg_buffer.size() >= Controller.BATCH_MAX_SIZE) ||
+                ((m.getType() != OFType.PACKET_OUT) && (m.getType() != OFType.FLOW_MOD))) {
+            this.write(msg_buffer);
+            msg_buffer.clear();
+        }
+    }
+
+    @Override
+    @LogMessageDoc(level = "WARN",
+            message = "Sending OF message that modifies switch " +
+                    "state while in the slave role: {switch}",
+            explanation = "An application has sent a message to a switch " +
+                    "that is not valid when the switch is in a slave role",
+            recommendation = LogMessageDoc.REPORT_CONTROLLER_BUG)
+    public void write(List<OFMessage> msglist,
+                      FloodlightContext bc) throws IOException {
+        for (OFMessage m : msglist) {
+            if (role == Role.SLAVE) {
+                switch (m.getType()) {
+                    case PACKET_OUT:
+                    case FLOW_MOD:
+                    case PORT_MOD:
+                        log.warn("Sending OF message that modifies switch " +
+                                "state while in the slave role: {}",
+                                m.getType().name());
+                        break;
+                    default:
+                        break;
+                }
+            }
+            // XXX S again
+            //this.floodlightProvider.handleOutgoingMessage(this, m, bc);
+        }
+        this.write(msglist);
+    }
+
+    public void write(List<OFMessage> msglist) throws IOException {
+        this.channel.write(msglist);
+    }
+
+    @Override
+    public void disconnectSwitch() {
+        channel.close();
+    }
+
+    @Override
+    public Date getConnectedSince() {
+        return connectedSince;
+    }
+
+    @JsonIgnore
+    @Override
+    public int getNextTransactionId() {
+        return this.transactionIdSource.incrementAndGet();
+    }
+
+    @JsonIgnore
+    @Override
+    public synchronized boolean isConnected() {
+        return connected;
+    }
+
+    @Override
+    @JsonIgnore
+    public synchronized void setConnected(boolean connected) {
+        this.connected = connected;
+    }
+
+
+
+    //*******************************************
+    //    Switch features related methods
+    //*******************************************
+
+    /**
+     * Set the features reply for this switch from the handshake
+     */
+    protected void setFeaturesReply(OFFeaturesReply featuresReply) {
+	if (featuresReply == null) {
+		log.error("Error setting featuresReply for switch: {}", getStringId());
+		return;
+	}
+	this.datapathId = featuresReply.getDatapathId();
+	this.capabilities = featuresReply.getCapabilities();
+	this.buffers = (int) featuresReply.getNBuffers();
+	this.tables = (byte) featuresReply.getNTables();
+	this.stringId = this.datapathId.toString();
+	if (ofversion == OFVersion.OF_13) {
+		auxId = featuresReply.getAuxiliaryId();
+		if (!auxId.equals(OFAuxId.MAIN)) {
+			log.warn("This controller does not handle auxiliary connections. "
+					+ "Aux connection id {} received from switch {}",
+					auxId, getStringId());
+		}
+	}
+
+	if (ofversion == OFVersion.OF_10) {
+		this.actions = featuresReply.getActions();
+		portManager.compareAndUpdatePorts(featuresReply.getPorts(), true);
+	}
+    }
+
+    /**
+     * Set the port descriptions for this switch from the handshake for
+     * an OF1.3 switch.
+     */
+    protected void setPortDescReply(OFPortDescStatsReply pdrep) {
+	if (ofversion != OFVersion.OF_13) return;
+	if (pdrep == null) {
+		log.error("Error setting ports description for switch: {}", getStringId());
+		return;
+	}
+	portManager.updatePorts(pdrep.getEntries());
+    }
+
+    @Override
+    public int getNumBuffers() {
+        return buffers;
+    }
+    @Override
+    public Set<OFActionType> getActions() {
+        return actions;
+    }
+    @Override
+    public  Set<OFCapabilities> getCapabilities() {
+        return capabilities;
+    }
+    @Override
+    public byte getNumTables() {
+        return tables;
+    }
+
+//    public Future<OFFeaturesReply> getFeaturesReplyFromSwitch()
+//            throws IOException {
+//    	// XXX S fix this later
+//    	OFMessage request = floodlightProvider.getOFMessageFactory_13()
+//    			.buildFeaturesRequest()
+//    			.setXid(getNextTransactionId())
+//    			.build();
+//    	OFFeaturesReplyFuture future =
+//                new OFFeaturesReplyFuture(threadPool, this, (int) request.getXid());
+//        this.featuresFutureMap.put((int) request.getXid(), future);
+//        this.channel.write(Collections.singletonList(request));
+//        return future;
+//
+//    }
+//
+//    public void deliverOFFeaturesReply(OFMessage reply) {
+//        OFFeaturesReplyFuture future = this.featuresFutureMap.get(reply.getXid());
+//        if (future != null) {
+//            future.deliverFuture(this, reply);
+//            // The future will ultimately unregister itself and call
+//            // cancelFeaturesReply
+//            return;
+//        }
+//        log.error("Switch {}: received unexpected featureReply", this);
+//    }
+
+    @Override
+    public void cancelFeaturesReply(int transactionId) {
+        this.featuresFutureMap.remove(transactionId);
+    }
+
+    @JsonIgnore
+    public void setSwitchDescription(OFDescStatsReply desc) {
+	switchDescription = desc;
+    }
+
+    @Override
+    @JsonIgnore
+    public OFDescStatsReply getSwitchDescription() {
+	return switchDescription;
+    }
+
+    //*******************************************
+    //    Switch port handling
+    //*******************************************
+
+    @Override
+    @JsonIgnore
+    public Collection<OFPortDesc> getEnabledPorts() {
+        return portManager.getEnabledPorts();
+    }
+
+    @Override
+    @JsonIgnore
+    public Collection<Integer> getEnabledPortNumbers() {
+        return portManager.getEnabledPortNumbers();
+    }
+
+    @Override
+    public OFPortDesc getPort(int portNumber) {
+        return portManager.getPort(portNumber);
+    }
+
+    @Override
+    public OFPortDesc getPort(String portName) {
+        return portManager.getPort(portName);
+    }
+
+    @Override
+    @JsonIgnore
+    public OrderedCollection<PortChangeEvent> processOFPortStatus(OFPortStatus ps) {
+        return portManager.handlePortStatusMessage(ps);
+    }
+
+    @Override
+    @JsonProperty("ports")
+    public Collection<OFPortDesc> getPorts() {
+	return portManager.getPorts();
+    }
+
+    @Override
+    public boolean portEnabled(int portNumber) {
+        return isEnabled(portManager.getPort(portNumber));
+    }
+
+    @Override
+    public boolean portEnabled(String portName) {
+	return isEnabled(portManager.getPort(portName));
+    }
+
+    private boolean isEnabled(OFPortDesc p) {
+	return (p != null &&
+			!p.getState().contains(OFPortState.LINK_DOWN) &&
+			!p.getState().contains(OFPortState.BLOCKED) &&
+                !p.getConfig().contains(OFPortConfig.PORT_DOWN));
+    }
+
+    @Override
+    public OrderedCollection<PortChangeEvent> comparePorts(Collection<OFPortDesc> ports) {
+        return portManager.comparePorts(ports);
+    }
+
+    @Override
+    @JsonIgnore
+    public OrderedCollection<PortChangeEvent> setPorts(Collection<OFPortDesc> ports) {
+        return portManager.updatePorts(ports);
+    }
+
+    /**
+     * Manages the ports of this switch.
+     *
+     * Provides methods to query and update the stored ports. The class ensures
+     * that every port name and port number is unique. When updating ports
+     * the class checks if port number <-> port name mappings have change due
+     * to the update. If a new port P has number and port that are inconsistent
+     * with the previous mapping(s) the class will delete all previous ports
+     * with name or number of the new port and then add the new port.
+     *
+     * Port names are stored as-is but they are compared case-insensitive
+     *
+     * The methods that change the stored ports return a list of
+     * PortChangeEvents that represent the changes that have been applied
+     * to the port list so that IOFSwitchListeners can be notified about the
+     * changes.
+     *
+     * Implementation notes:
+     * - We keep several different representations of the ports to allow for
+     *   fast lookups
+     * - Ports are stored in unchangeable lists. When a port is modified new
+     *   data structures are allocated.
+     * - We use a read-write-lock for synchronization, so multiple readers are
+     *   allowed.
+     * - All port numbers have int representation (no more shorts)
+     */
+    protected class PortManager {
+        private final ReentrantReadWriteLock lock;
+        private List<OFPortDesc> portList;
+        private List<OFPortDesc> enabledPortList;
+        private List<Integer> enabledPortNumbers;
+        private Map<Integer,OFPortDesc> portsByNumber;
+        private Map<String,OFPortDesc> portsByName;
+
+        public PortManager() {
+            this.lock = new ReentrantReadWriteLock();
+            this.portList = Collections.emptyList();
+            this.enabledPortList = Collections.emptyList();
+            this.enabledPortNumbers = Collections.emptyList();
+            this.portsByName = Collections.emptyMap();
+            this.portsByNumber = Collections.emptyMap();
+        }
+
+        /**
+         * Set the internal data structure storing this switch's port
+         * to the ports specified by newPortsByNumber
+         *
+         * CALLER MUST HOLD WRITELOCK
+         *
+         * @param newPortsByNumber
+         * @throws IllegaalStateException if called without holding the
+         * writelock
+         */
+        private void updatePortsWithNewPortsByNumber(
+                Map<Integer,OFPortDesc> newPortsByNumber) {
+            if (!lock.writeLock().isHeldByCurrentThread()) {
+                throw new IllegalStateException("Method called without " +
+                                                "holding writeLock");
+            }
+            Map<String,OFPortDesc> newPortsByName =
+                    new HashMap<String, OFPortDesc>();
+            List<OFPortDesc> newPortList =
+                    new ArrayList<OFPortDesc>();
+            List<OFPortDesc> newEnabledPortList =
+                    new ArrayList<OFPortDesc>();
+            List<Integer> newEnabledPortNumbers = new ArrayList<Integer>();
+
+            for(OFPortDesc p: newPortsByNumber.values()) {
+                newPortList.add(p);
+                newPortsByName.put(p.getName().toLowerCase(), p);
+                if (isEnabled(p)) {
+                    newEnabledPortList.add(p);
+                    newEnabledPortNumbers.add(p.getPortNo().getPortNumber());
+                }
+            }
+            portsByName = Collections.unmodifiableMap(newPortsByName);
+            portsByNumber =
+                    Collections.unmodifiableMap(newPortsByNumber);
+            enabledPortList =
+                    Collections.unmodifiableList(newEnabledPortList);
+            enabledPortNumbers =
+                    Collections.unmodifiableList(newEnabledPortNumbers);
+            portList = Collections.unmodifiableList(newPortList);
+        }
+
+        /**
+         * Handle a OFPortStatus delete message for the given port.
+         * Updates the internal port maps/lists of this switch and returns
+         * the PortChangeEvents caused by the delete. If the given port
+         * exists as it, it will be deleted. If the name<->number for the
+         * given port is inconsistent with the ports stored by this switch
+         * the method will delete all ports with the number or name of the
+         * given port.
+         *
+         * This method will increment error/warn counters and log
+         *
+         * @param delPort the port from the port status message that should
+         * be deleted.
+         * @return ordered collection of port changes applied to this switch
+         */
+        private OrderedCollection<PortChangeEvent> handlePortStatusDelete(
+				OFPortDesc delPort) {
+            lock.writeLock().lock();
+            OrderedCollection<PortChangeEvent> events =
+                    new LinkedHashSetWrapper<PortChangeEvent>();
+            try {
+                Map<Integer,OFPortDesc> newPortByNumber =
+                        new HashMap<Integer, OFPortDesc>(portsByNumber);
+                OFPortDesc prevPort =
+                        portsByNumber.get(delPort.getPortNo().getPortNumber());
+                if (prevPort == null) {
+                    // so such port. Do we have a port with the name?
+                    prevPort = portsByName.get(delPort.getName());
+                    if (prevPort != null) {
+                        newPortByNumber.remove(prevPort.getPortNo().getPortNumber());
+                        events.add(new PortChangeEvent(prevPort,
+                                                       PortChangeType.DELETE));
+                    }
+                } else if (prevPort.getName().equals(delPort.getName())) {
+                    // port exists with consistent name-number mapping
+                    newPortByNumber.remove(delPort.getPortNo().getPortNumber());
+                    events.add(new PortChangeEvent(delPort,
+                                                   PortChangeType.DELETE));
+                } else {
+                    // port with same number exists but its name differs. This
+                    // is weird. The best we can do is to delete the existing
+                    // port(s) that have delPort's name and number.
+                    newPortByNumber.remove(delPort.getPortNo().getPortNumber());
+                    events.add(new PortChangeEvent(prevPort,
+                                                   PortChangeType.DELETE));
+                    // is there another port that has delPort's name?
+                    prevPort = portsByName.get(delPort.getName().toLowerCase());
+                    if (prevPort != null) {
+                        newPortByNumber.remove(prevPort.getPortNo().getPortNumber());
+                        events.add(new PortChangeEvent(prevPort,
+                                                       PortChangeType.DELETE));
+                    }
+                }
+                updatePortsWithNewPortsByNumber(newPortByNumber);
+                return events;
+            } finally {
+                lock.writeLock().unlock();
+            }
+        }
+
+        /**
+         * Handle a OFPortStatus message, update the internal data structures
+         * that store ports and return the list of OFChangeEvents.
+         *
+         * This method will increment error/warn counters and log
+         *
+         * @param ps
+         * @return
+         */
+        public OrderedCollection<PortChangeEvent> handlePortStatusMessage(OFPortStatus ps) {
+            if (ps == null) {
+                throw new NullPointerException("OFPortStatus message must " +
+                                               "not be null");
+            }
+            lock.writeLock().lock();
+            try {
+                OFPortReason reason = ps.getReason();
+                if (reason == null) {
+                    throw new IllegalArgumentException("Unknown PortStatus " +
+                            "reason code " + ps.getReason());
+                }
+
+                if (log.isDebugEnabled()) {
+                    log.debug("Handling OFPortStatus: {} for {}",
+                              reason, ps);
+                }
+
+                if (reason == OFPortReason.DELETE)
+                        return handlePortStatusDelete(ps.getDesc());
+
+                // We handle ADD and MODIFY the same way. Since OpenFlow
+                // doesn't specify what uniquely identifies a port the
+                // notion of ADD vs. MODIFY can also be hazy. So we just
+                // compare the new port to the existing ones.
+                Map<Integer,OFPortDesc> newPortByNumber =
+                    new HashMap<Integer, OFPortDesc>(portsByNumber);
+                OrderedCollection<PortChangeEvent> events =
+				getSinglePortChanges(ps.getDesc());
+                for (PortChangeEvent e: events) {
+                    switch(e.type) {
+                        case DELETE:
+                            newPortByNumber.remove(e.port.getPortNo().getPortNumber());
+                            break;
+                        case ADD:
+                            if (reason != OFPortReason.ADD) {
+                                // weird case
+                            }
+                            newPortByNumber.put(e.port.getPortNo().getPortNumber(),
+                                    e.port);
+                            break;
+                        case DOWN:
+                            newPortByNumber.put(e.port.getPortNo().getPortNumber(),
+                                    e.port);
+                            break;
+                        case OTHER_UPDATE:
+                            newPortByNumber.put(e.port.getPortNo().getPortNumber(),
+                                    e.port);
+                            break;
+                        case UP:
+                            newPortByNumber.put(e.port.getPortNo().getPortNumber(),
+					e.port);
+                            break;
+                    }
+                }
+                updatePortsWithNewPortsByNumber(newPortByNumber);
+                return events;
+            } finally {
+                lock.writeLock().unlock();
+            }
+
+        }
+
+        /**
+         * Given a new or modified port newPort, returns the list of
+         * PortChangeEvents to "transform" the current ports stored by
+         * this switch to include / represent the new port. The ports stored
+         * by this switch are <b>NOT</b> updated.
+         *
+         * This method acquires the readlock and is thread-safe by itself.
+         * Most callers will need to acquire the write lock before calling
+         * this method though (if the caller wants to update the ports stored
+         * by this switch)
+         *
+         * @param newPort the new or modified port.
+         * @return the list of changes
+         */
+        public OrderedCollection<PortChangeEvent> getSinglePortChanges(
+			OFPortDesc newPort) {
+            lock.readLock().lock();
+            try {
+                OrderedCollection<PortChangeEvent> events =
+                        new LinkedHashSetWrapper<PortChangeEvent>();
+                // Check if we have a port by the same number in our
+                // old map.
+                OFPortDesc prevPort =
+                        portsByNumber.get(newPort.getPortNo().getPortNumber());
+                if (newPort.equals(prevPort)) {
+                    // nothing has changed
+                    return events;
+                }
+
+                if (prevPort != null &&
+                        prevPort.getName().equals(newPort.getName())) {
+                    // A simple modify of a existing port
+                    // A previous port with this number exists and it's name
+                    // also matches the new port. Find the differences
+                    if (isEnabled(prevPort) && !isEnabled(newPort)) {
+                        events.add(new PortChangeEvent(newPort,
+                                                       PortChangeType.DOWN));
+                    } else if (!isEnabled(prevPort) && isEnabled(newPort)) {
+                        events.add(new PortChangeEvent(newPort,
+                                                       PortChangeType.UP));
+                    } else {
+                        events.add(new PortChangeEvent(newPort,
+                                   PortChangeType.OTHER_UPDATE));
+                    }
+                    return events;
+                }
+
+                if (prevPort != null) {
+                    // There exists a previous port with the same port
+                    // number but the port name is different (otherwise we would
+                    // never have gotten here)
+                    // Remove the port. Name-number mapping(s) have changed
+                    events.add(new PortChangeEvent(prevPort,
+                                                   PortChangeType.DELETE));
+                }
+
+                // We now need to check if there exists a previous port sharing
+                // the same name as the new/updated port.
+                prevPort = portsByName.get(newPort.getName().toLowerCase());
+                if (prevPort != null) {
+                    // There exists a previous port with the same port
+                    // name but the port number is different (otherwise we
+                    // never have gotten here).
+                    // Remove the port. Name-number mapping(s) have changed
+                    events.add(new PortChangeEvent(prevPort,
+                                                   PortChangeType.DELETE));
+                }
+
+                // We always need to add the new port. Either no previous port
+                // existed or we just deleted previous ports with inconsistent
+                // name-number mappings
+                events.add(new PortChangeEvent(newPort, PortChangeType.ADD));
+                return events;
+            } finally {
+                lock.readLock().unlock();
+            }
+        }
+
+        /**
+         * Compare the current ports of this switch to the newPorts list and
+         * return the changes that would be applied to transfort the current
+         * ports to the new ports. No internal data structures are updated
+         * see {@link #compareAndUpdatePorts(List, boolean)}
+         *
+         * @param newPorts the list of new ports
+         * @return The list of differences between the current ports and
+         * newPortList
+         */
+        public OrderedCollection<PortChangeEvent> comparePorts(
+			Collection<OFPortDesc> newPorts) {
+            return compareAndUpdatePorts(newPorts, false);
+        }
+
+        /**
+         * Compare the current ports of this switch to the newPorts list and
+         * return the changes that would be applied to transform the current
+         * ports to the new ports. No internal data structures are updated
+         * see {@link #compareAndUpdatePorts(List, boolean)}
+         *
+         * @param newPorts the list of new ports
+         * @return The list of differences between the current ports and
+         * newPortList
+         */
+        public OrderedCollection<PortChangeEvent> updatePorts(
+			Collection<OFPortDesc> newPorts) {
+            return compareAndUpdatePorts(newPorts, true);
+        }
+
+        /**
+         * Compare the current ports stored in this switch instance with the
+         * new port list given and return the differences in the form of
+         * PortChangeEvents. If the doUpdate flag is true, newPortList will
+         * replace the current list of this switch (and update the port maps)
+         *
+         * Implementation note:
+         * Since this method can optionally modify the current ports and
+         * since it's not possible to upgrade a read-lock to a write-lock
+         * we need to hold the write-lock for the entire operation. If this
+         * becomes a problem and if compares() are common we can consider
+         * splitting in two methods but this requires lots of code duplication
+         *
+         * @param newPorts the list of new ports.
+         * @param doUpdate If true the newPortList will replace the current
+         * port list for this switch. If false this switch will not be changed.
+         * @return The list of differences between the current ports and
+         * newPorts
+         * @throws NullPointerException if newPortsList is null
+         * @throws IllegalArgumentException if either port names or port numbers
+         * are duplicated in the newPortsList.
+         */
+        private OrderedCollection<PortChangeEvent> compareAndUpdatePorts(
+                Collection<OFPortDesc> newPorts, boolean doUpdate) {
+            if (newPorts == null) {
+                throw new NullPointerException("newPortsList must not be null");
+            }
+            lock.writeLock().lock();
+            try {
+                OrderedCollection<PortChangeEvent> events =
+                        new LinkedHashSetWrapper<PortChangeEvent>();
+
+                Map<Integer,OFPortDesc> newPortsByNumber =
+                        new HashMap<Integer, OFPortDesc>();
+                Map<String,OFPortDesc> newPortsByName =
+                        new HashMap<String, OFPortDesc>();
+                List<OFPortDesc> newEnabledPortList =
+                        new ArrayList<OFPortDesc>();
+                List<Integer> newEnabledPortNumbers =
+                        new ArrayList<Integer>();
+                List<OFPortDesc> newPortsList =
+                        new ArrayList<OFPortDesc>(newPorts);
+
+                for (OFPortDesc p: newPortsList) {
+                    if (p == null) {
+                        throw new NullPointerException("portList must not " +
+                                "contain null values");
+                    }
+
+                    // Add the port to the new maps and lists and check
+                    // that every port is unique
+                    OFPortDesc duplicatePort;
+                    duplicatePort = newPortsByNumber.put(
+				p.getPortNo().getPortNumber(), p);
+                    if (duplicatePort != null) {
+                        String msg = String.format("Cannot have two ports " +
+                                "with the same number: %s <-> %s",
+                                p, duplicatePort);
+                        throw new IllegalArgumentException(msg);
+                    }
+                    duplicatePort =
+                            newPortsByName.put(p.getName().toLowerCase(), p);
+                    if (duplicatePort != null) {
+                        String msg = String.format("Cannot have two ports " +
+                                "with the same name: %s <-> %s",
+                                p.toString().substring(0,80),
+                                duplicatePort.toString().substring(0, 80));
+                        throw new IllegalArgumentException(msg);
+                    }
+                    if (isEnabled(p)) {
+                        newEnabledPortList.add(p);
+                        newEnabledPortNumbers.add(p.getPortNo().getPortNumber());
+                    }
+
+                    // get changes
+                    events.addAll(getSinglePortChanges(p));
+                }
+                // find deleted ports
+                // We need to do this after looping through all the new ports
+                // to we can handle changed name<->number mappings correctly
+                // We could pull it into the loop of we address this but
+                // it's probably not worth it
+                for (OFPortDesc oldPort: this.portList) {
+                    if (!newPortsByNumber.containsKey(oldPort.getPortNo().getPortNumber())) {
+                        PortChangeEvent ev =
+                                new PortChangeEvent(oldPort,
+                                                    PortChangeType.DELETE);
+                        events.add(ev);
+                    }
+                }
+
+
+                if (doUpdate) {
+                    portsByName = Collections.unmodifiableMap(newPortsByName);
+                    portsByNumber =
+                            Collections.unmodifiableMap(newPortsByNumber);
+                    enabledPortList =
+                            Collections.unmodifiableList(newEnabledPortList);
+                    enabledPortNumbers =
+                            Collections.unmodifiableList(newEnabledPortNumbers);
+                    portList = Collections.unmodifiableList(newPortsList);
+                }
+                return events;
+            } finally {
+                lock.writeLock().unlock();
+            }
+        }
+
+        public OFPortDesc getPort(String name) {
+            if (name == null) {
+                throw new NullPointerException("Port name must not be null");
+            }
+            lock.readLock().lock();
+            try {
+                return portsByName.get(name.toLowerCase());
+            } finally {
+                lock.readLock().unlock();
+            }
+        }
+
+        public OFPortDesc getPort(int portNumber) {
+            lock.readLock().lock();
+            try {
+                return portsByNumber.get(portNumber);
+            } finally {
+                lock.readLock().unlock();
+            }
+        }
+
+        public List<OFPortDesc> getPorts() {
+            lock.readLock().lock();
+            try {
+                return portList;
+            } finally {
+                lock.readLock().unlock();
+            }
+        }
+
+        public List<OFPortDesc> getEnabledPorts() {
+            lock.readLock().lock();
+            try {
+                return enabledPortList;
+            } finally {
+                lock.readLock().unlock();
+            }
+        }
+
+        public List<Integer> getEnabledPortNumbers() {
+            lock.readLock().lock();
+            try {
+                return enabledPortNumbers;
+            } finally {
+                lock.readLock().unlock();
+            }
+        }
+    }
+
+
+	//*******************************************
+    //    Switch stats handling
+    //*******************************************
+
+    @Override
+    public Future<List<OFStatsReply>> getStatistics(OFStatsRequest<?> request)
+		throws IOException {
+        OFStatisticsFuture future = new OFStatisticsFuture(threadPool, this,
+			(int)request.getXid());
+        log.info("description STAT REQUEST XID {}", request.getXid());
+        this.statsFutureMap.put((int)request.getXid(), future);
+
+        this.channel.write(Collections.singletonList(request));
+        return future;
+    }
+
+    @Override
+    public void deliverStatisticsReply(OFMessage reply) {
+        OFStatisticsFuture future = this.statsFutureMap.get((int)reply.getXid());
+        if (future != null) {
+            future.deliverFuture(this, reply);
+            // The future will ultimately unregister itself and call
+            // cancelStatisticsReply
+            return;
+        }
+        // Transaction id was not found in statsFutureMap.check the other map
+        IOFMessageListener caller = this.iofMsgListenersMap.get(reply.getXid());
+        if (caller != null) {
+		caller.receive(this, reply, null);
+        }
+    }
+
+    @Override
+    public void cancelStatisticsReply(int transactionId) {
+        if (null == this.statsFutureMap.remove(transactionId)) {
+            this.iofMsgListenersMap.remove(transactionId);
+        }
+    }
+
+    @Override
+    public void cancelAllStatisticsReplies() {
+        /* we don't need to be synchronized here. Even if another thread
+         * modifies the map while we're cleaning up the future will eventuall
+         * timeout */
+        for (OFStatisticsFuture f : statsFutureMap.values()) {
+            f.cancel(true);
+        }
+        statsFutureMap.clear();
+        iofMsgListenersMap.clear();
+    }
+
+
+    //*******************************************
+    //    Switch role handling
+    //*******************************************
+
+    // XXX S this is completely wrong. The generation id should be obtained
+    // after coordinating with all controllers connected to this switch.
+    // ie. should be part of registry service (account for 1.3 vs 1.0)
+    // For now we are just generating this locally and keeping it constant.
+    public U64 getNextGenerationId() {
+        //TODO: Pankaj, fix nextGenerationId as part of Registry interface
+	return U64.of(generationIdSource);
+    }
+
+    @Override
+    public Role getRole() {
+        return role;
+    }
+
+    @JsonIgnore
+    @Override
+    public void setRole(Role role) {
+        this.role = role;
+    }
+
+
+    //*******************************************
+    //    Switch utility methods
+    //*******************************************
+
+    private void registerOverloadCounters() throws CounterException {
+        if (debugCountersRegistered) {
+            return;
+        }
+        if (debugCounters == null) {
+            log.error("Debug Counter Service not found");
+            debugCounters = new NullDebugCounter();
+            debugCountersRegistered = true;
+        }
+        // every level of the hierarchical counter has to be registered
+        // even if they are not used
+        ctrSwitch = debugCounters.registerCounter(
+                                   BASE , stringId,
+                                   "Counter for this switch",
+                                   CounterType.ALWAYS_COUNT);
+        ctrSwitchPktin = debugCounters.registerCounter(
+                                   BASE, stringId + "/pktin",
+                                   "Packet in counter for this switch",
+                                   CounterType.ALWAYS_COUNT);
+        ctrSwitchWrite = debugCounters.registerCounter(
+                                   BASE, stringId + "/write",
+                                   "Write counter for this switch",
+                                   CounterType.ALWAYS_COUNT);
+        ctrSwitchPktinDrops = debugCounters.registerCounter(
+                                   BASE, stringId + "/pktin/drops",
+                                   "Packet in throttle drop count",
+                                   CounterType.ALWAYS_COUNT,
+                                   IDebugCounterService.CTR_MDATA_WARN);
+        ctrSwitchWriteDrops = debugCounters.registerCounter(
+                                   BASE, stringId + "/write/drops",
+                                   "Switch write throttle drop count",
+                                   CounterType.ALWAYS_COUNT,
+                                   IDebugCounterService.CTR_MDATA_WARN);
+    }
+
+    @Override
+    public void startDriverHandshake() throws IOException {
+	log.debug("Starting driver handshake for sw {}", getStringId());
+        if (startDriverHandshakeCalled)
+            throw new SwitchDriverSubHandshakeAlreadyStarted();
+        startDriverHandshakeCalled = true;
+    }
+
+    @Override
+    public boolean isDriverHandshakeComplete() {
+        if (!startDriverHandshakeCalled)
+            throw new SwitchDriverSubHandshakeNotStarted();
+        return true;
+    }
+
+    @Override
+    public void processDriverHandshakeMessage(OFMessage m) {
+        if (startDriverHandshakeCalled)
+            throw new SwitchDriverSubHandshakeCompleted(m);
+        else
+            throw new SwitchDriverSubHandshakeNotStarted();
+    }
+
+    @Override
+    @JsonIgnore
+    @LogMessageDoc(level="WARN",
+        message="Switch {switch} flow table is full",
+        explanation="The controller received flow table full " +
+                "message from the switch, could be caused by increased " +
+                "traffic pattern",
+                recommendation=LogMessageDoc.REPORT_CONTROLLER_BUG)
+    public void setTableFull(boolean isFull) {
+        if (isFull && !flowTableFull) {
+            floodlightProvider.addSwitchEvent(this.datapathId.getLong(),
+                    "SWITCH_FLOW_TABLE_FULL " +
+                    "Table full error from switch", false);
+            log.warn("Switch {} flow table is full", stringId);
+        }
+        flowTableFull  = isFull;
+    }
+
+    @Override
+    @LogMessageDoc(level = "ERROR",
+            message = "Failed to clear all flows on switch {switch}",
+            explanation = "An I/O error occured while trying to clear " +
+                    "flows on the switch.",
+            recommendation = LogMessageDoc.CHECK_SWITCH)
+    public void clearAllFlowMods() {
+        // Delete all pre-existing flows
+
+	// by default if match is not specified, then an empty list of matches
+	// is sent, resulting in a wildcard-all flows
+	// XXX fix this later
+	OFMessage fm = floodlightProvider.getOFMessageFactory_13()
+						.buildFlowDelete()
+						.setOutPort(OFPort.ANY)
+						.setOutGroup(OFGroup.ANY)
+						.setTableId(TableId.ALL)
+						.build();
+
+        try {
+            channel.write(Collections.singletonList(fm));
+        } catch (Exception e) {
+            log.error("Failed to clear all flows on switch " + this, e);
+        }
+    }
+
+    @Override
+    public void flush() {
+        Map<OFSwitchImplBase, List<OFMessage>> msg_buffer_map = local_msg_buffer.get();
+        List<OFMessage> msglist = msg_buffer_map.get(this);
+        if ((msglist != null) && (msglist.size() > 0)) {
+            try {
+                this.write(msglist);
+            } catch (IOException e) {
+		log.error("Failed flushing messages", e);
+            }
+            msglist.clear();
+        }
+    }
+
+    public static void flush_all() {
+        Map<OFSwitchImplBase, List<OFMessage>> msg_buffer_map = local_msg_buffer.get();
+        for (OFSwitchImplBase sw : msg_buffer_map.keySet()) {
+            sw.flush();
+        }
+    }
+
+    /**
+     * Return a read lock that must be held while calling the listeners for
+     * messages from the switch. Holding the read lock prevents the active
+     * switch list from being modified out from under the listeners.
+     *
+     * @return listener read lock
+     */
+    @JsonIgnore
+    public Lock getListenerReadLock() {
+        return listenerLock.readLock();
+    }
+
+    /**
+     * Return a write lock that must be held when the controllers modifies the
+     * list of active switches. This is to ensure that the active switch list
+     * doesn't change out from under the listeners as they are handling a
+     * message from the switch.
+     *
+     * @return listener write lock
+     */
+    @JsonIgnore
+    public Lock getListenerWriteLock() {
+        return listenerLock.writeLock();
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/core/internal/OpenflowPipelineFactory.java b/src/main/java/net/floodlightcontroller/core/internal/OpenflowPipelineFactory.java
index e87989d..62938fe 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/OpenflowPipelineFactory.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/OpenflowPipelineFactory.java
@@ -1,19 +1,19 @@
 /**
- *    Copyright 2011, Big Switch Networks, Inc.
- *    Originally created by David Erickson, Stanford University
- *
- *    Licensed under the Apache License, Version 2.0 (the "License"); you may
- *    not use this file except in compliance with the License. You may obtain
- *    a copy of the License at
- *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- *    License for the specific language governing permissions and limitations
- *    under the License.
- **/
+*    Copyright 2011, Big Switch Networks, Inc.
+*    Originally created by David Erickson, Stanford University
+*
+*    Licensed under the Apache License, Version 2.0 (the "License"); you may
+*    not use this file except in compliance with the License. You may obtain
+*    a copy of the License at
+*
+*         http://www.apache.org/licenses/LICENSE-2.0
+*
+*    Unless required by applicable law or agreed to in writing, software
+*    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+*    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+*    License for the specific language governing permissions and limitations
+*    under the License.
+**/
 
 package net.floodlightcontroller.core.internal;
 
@@ -25,15 +25,16 @@
 import org.jboss.netty.handler.execution.ExecutionHandler;
 import org.jboss.netty.handler.timeout.IdleStateHandler;
 import org.jboss.netty.handler.timeout.ReadTimeoutHandler;
+import org.jboss.netty.util.ExternalResourceReleasable;
 import org.jboss.netty.util.HashedWheelTimer;
 import org.jboss.netty.util.Timer;
 
 /**
  * Creates a ChannelPipeline for a server-side openflow channel
- *
  * @author readams
  */
-public class OpenflowPipelineFactory implements ChannelPipelineFactory {
+public class OpenflowPipelineFactory
+    implements ChannelPipelineFactory, ExternalResourceReleasable {
 
     protected Controller controller;
     protected ThreadPoolExecutor pipelineExecutor;
@@ -53,20 +54,25 @@
 
     @Override
     public ChannelPipeline getPipeline() throws Exception {
-        OFChannelState state = new OFChannelState();
+        OFChannelHandler handler = new OFChannelHandler(controller);
 
         ChannelPipeline pipeline = Channels.pipeline();
         pipeline.addLast("ofmessagedecoder", new OFMessageDecoder());
         pipeline.addLast("ofmessageencoder", new OFMessageEncoder());
         pipeline.addLast("idle", idleHandler);
         pipeline.addLast("timeout", readTimeoutHandler);
+        // XXX S ONOS: was 15 increased it to fix Issue #296
         pipeline.addLast("handshaketimeout",
-                new HandshakeTimeoutHandler(state, timer, 60)); // ONOS: was 15 increased it to fix Issue #296
+                         new HandshakeTimeoutHandler(handler, timer, 60));
         if (pipelineExecutor != null)
             pipeline.addLast("pipelineExecutor",
-                    new ExecutionHandler(pipelineExecutor));
-        pipeline.addLast("handler", controller.getChannelHandler(state));
+                             new ExecutionHandler(pipelineExecutor));
+        pipeline.addLast("handler", handler);
         return pipeline;
     }
 
+    @Override
+    public void releaseExternalResources() {
+        timer.stop();
+    }
 }
diff --git a/src/main/java/net/floodlightcontroller/core/internal/RoleChanger.java b/src/main/java/net/floodlightcontroller/core/internal/RoleChanger.java
deleted file mode 100644
index 6252a72..0000000
--- a/src/main/java/net/floodlightcontroller/core/internal/RoleChanger.java
+++ /dev/null
@@ -1,342 +0,0 @@
-package net.floodlightcontroller.core.internal;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.concurrent.DelayQueue;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.TimeUnit;
-
-import net.floodlightcontroller.core.IFloodlightProviderService.Role;
-import net.floodlightcontroller.core.IOFSwitch;
-import net.floodlightcontroller.core.annotations.LogMessageDoc;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class handles sending of RoleRequest messages to all connected switches.
- * <p/>
- * Handling Role Requests is tricky. Roles are hard state on the switch and
- * we can't query it so we need to make sure that we have consistent states
- * on the switches. Whenever we send a role request to the set of connected
- * switches we need to make sure that we've sent the request to all of them
- * before we process the next change request. If a new switch connects, we
- * need to send it the current role and need to make sure that the current
- * role doesn't change while we are doing it. We achieve this by synchronizing
- * all these actions on Controller.roleChanger
- * On the receive side: we need to make sure that we receive a reply for each
- * request we send and that the reply is consistent with the request we sent.
- * We'd also like to send the role request to the switch asynchronously in a
- * separate thread so we don't block the REST API or other callers.
- * <p/>
- * There are potential ways to relax these synchronization requirements:
- * - "Generation ID" for each role request. However, this would be most useful
- * if it were global for the whole cluster
- * - Regularly resend the controller's current role. Don't know whether this
- * might have adverse effects on the switch.
- * <p/>
- * Caveats:
- * - No way to know if another controller (not in our controller cluster)
- * sends MASTER requests to connected switches. Then we would drop to
- * slave role without knowing it. Could regularly resend the current role.
- * Ideally the switch would notify us if it demoted us. What happens if
- * the other controller also regularly resends the same role request?
- * Or if the health check determines that
- * a controller is dead but the controller is still talking to switches (maybe
- * just its health check failed) and resending the master role request....
- * We could try to detect if a switch demoted us to slave even if we think
- * we are master (error messages on packet outs, e.g., when sending LLDPs)
- * <p/>
- * <p/>
- * The general model of Role Request handling is as follows:
- * <p/>
- * - All role request messages are handled by this class. Class Controller
- * submits a role change request and the request gets queued. submitRequest
- * takes a Collection of switches to which to send the request. We make a copy
- * of this list.
- * - A thread takes these change requests from the queue and sends them to
- * all the switches (using our copy of the switch list).
- * - The OFSwitchImpl sends the request over the wire and puts the request
- * into a queue of pending request (storing xid and role). We start a timeout
- * to make sure we eventually receive a reply from the switch. We use a single
- * timeout for each request submitted using submitRequest()
- * - After the timeout triggers we go over the list of switches again and
- * check that a response has been received (by checking the head of the
- * OFSwitchImpl's queue of pending requests)
- * - We handle requests and timeouts in the same thread. We use a priority queue
- * to schedule them so we are guaranteed that they are processed in
- * the same order as they are submitted. If a request times out we drop
- * the connection to this switch.
- * - Since we decouple submission of role change requests and actually sending
- * them we cannot check a received role reply against the controller's current
- * role because the controller's current role could have changed again.
- * - Receiving Role Reply messages is handled by OFChannelHandler and
- * OFSwitchImpl directly. The OFSwitchImpl checks if the received request
- * is as expected (xid and role match the head of the pending queue in
- * OFSwitchImpl). If so
- * the switch updates its role. Otherwise the connection is dropped. If this
- * is the first reply, the SWITCH_SUPPORTS_NX_ROLE attribute is set.
- * Next, we call addSwitch(), removeSwitch() to update the list of active
- * switches if appropriate.
- * - If we receive an Error indicating that roles are not supported by the
- * switch, we set the SWITCH_SUPPORTS_NX_ROLE to false. We keep the
- * switch connection alive while in MASTER and EQUAL role.
- * (TODO: is this the right behavior for EQUAL??). If the role changes to
- * SLAVE the switch connection is dropped (remember: only if the switch
- * doesn't support role requests)
- * The expected behavior is that the switch will probably try to reconnect
- * repeatedly (with some sort of exponential backoff), but after a  while
- * will give-up and move on to the next controller-IP configured on the
- * switch. This is the serial failover mechanism from OpenFlow spec v1.0.
- * <p/>
- * New switch connection:
- * - Switch handshake is done without sending any role request messages.
- * - After handshake completes, switch is added to the list of connected switches
- * and we send the first role request message if role
- * requests are enabled. If roles are disabled automatically promote switch to
- * active switch list and clear FlowTable.
- * - When we receive the first reply we proceed as above. In addition, if
- * the role request is for MASTER we wipe the flow table. We do not wipe
- * the flow table if the switch connected while role supported was disabled
- * on the controller.
- */
-public class RoleChanger {
-    // FIXME: Upon closer inspection DelayQueue seems to be somewhat broken. 
-    // We are required to implement a compareTo based on getDelay() and 
-    // getDelay() must return the remaining delay, thus it needs to use the 
-    // current time. So x1.compareTo(x1) can never return 0 as some time
-    // will have passed between evaluating both getDelays(). This is even worse
-    // if the thread happens to be preempted between calling the getDelay()
-    // For the time being we enforce a small delay between subsequent
-    // role request messages and hope that's long enough to not screw up
-    // ordering. In the long run we might want to use two threads and two queues
-    // (one for requests, one for timeouts)
-    // Sigh. 
-    protected DelayQueue<RoleChangeTask> pendingTasks;
-    protected long lastSubmitTime;
-    protected Thread workerThread;
-    protected long timeout;
-    protected final static long DEFAULT_TIMEOUT = 15L * 1000 * 1000 * 1000L; // 15s
-    protected final static Logger log = LoggerFactory.getLogger(RoleChanger.class);
-
-    /**
-     * A queued task to be handled by the Role changer thread.
-     */
-    protected static class RoleChangeTask implements Delayed {
-        protected enum Type {
-            /**
-             * This is a request. Dispatch the role update to switches
-             */
-            REQUEST,
-            /**
-             * This is a timeout task. Check if all switches have
-             * correctly replied to the previously dispatched role request
-             */
-            TIMEOUT
-        }
-
-        // The set of switches to work on
-        public Collection<OFSwitchImpl> switches;
-        public Role role;
-        public Type type;
-        // the time when the task should run as nanoTime() 
-        public long deadline;
-
-        public RoleChangeTask(Collection<OFSwitchImpl> switches, Role role, long deadline) {
-            this.switches = switches;
-            this.role = role;
-            this.type = Type.REQUEST;
-            this.deadline = deadline;
-        }
-
-        @Override
-        public int compareTo(Delayed o) {
-            Long timeRemaining = getDelay(TimeUnit.NANOSECONDS);
-            return timeRemaining.compareTo(o.getDelay(TimeUnit.NANOSECONDS));
-        }
-
-        @Override
-        public long getDelay(TimeUnit tu) {
-            long timeRemaining = deadline - System.nanoTime();
-            return tu.convert(timeRemaining, TimeUnit.NANOSECONDS);
-        }
-    }
-
-    @LogMessageDoc(level = "ERROR",
-            message = "RoleRequestWorker task had an uncaught exception.",
-            explanation = "An unknown occured while processing an HA " +
-                    "role change event.",
-            recommendation = LogMessageDoc.GENERIC_ACTION)
-    protected class RoleRequestWorker extends Thread {
-        @Override
-        public void run() {
-            RoleChangeTask t;
-            boolean interrupted = false;
-            log.trace("RoleRequestWorker thread started");
-            try {
-                while (true) {
-                    try {
-                        t = pendingTasks.take();
-                    } catch (InterruptedException e) {
-                        // see http://www.ibm.com/developerworks/java/library/j-jtp05236/index.html
-                        interrupted = true;
-                        continue;
-                    }
-                    if (t.type == RoleChangeTask.Type.REQUEST) {
-                        sendRoleRequest(t.switches, t.role, t.deadline);
-                        // Queue the timeout
-                        t.type = RoleChangeTask.Type.TIMEOUT;
-                        t.deadline += timeout;
-                        pendingTasks.put(t);
-                    } else {
-                        verifyRoleReplyReceived(t.switches, t.deadline);
-                    }
-                }
-            } catch (Exception e) {
-                // Should never get here
-                log.error("RoleRequestWorker task had an uncaught exception. ",
-                        e);
-            } finally {
-                // Be nice in case we earlier caught InterruptedExecution
-                if (interrupted)
-                    Thread.currentThread().interrupt();
-            }
-        } // end loop
-    }
-
-    public RoleChanger() {
-        this.pendingTasks = new DelayQueue<RoleChangeTask>();
-        this.workerThread = new Thread(new RoleRequestWorker());
-        this.timeout = DEFAULT_TIMEOUT;
-        this.workerThread.start();
-    }
-
-
-    public synchronized void submitRequest(Collection<OFSwitchImpl> switches, Role role) {
-        long deadline = System.nanoTime();
-        // Grrr. stupid DelayQueue. Make sre we have at least 10ms between 
-        // role request messages.
-        if (deadline - lastSubmitTime < 10 * 1000 * 1000)
-            deadline = lastSubmitTime + 10 * 1000 * 1000;
-        // make a copy of the list 
-        ArrayList<OFSwitchImpl> switches_copy = new ArrayList<OFSwitchImpl>(switches);
-        RoleChangeTask req = new RoleChangeTask(switches_copy, role, deadline);
-        pendingTasks.put(req);
-        lastSubmitTime = deadline;
-    }
-
-    /**
-     * Send a role request message to switches. This checks the capabilities
-     * of the switch for understanding role request messaging. Currently we only
-     * support the OVS-style role request message, but once the controller
-     * supports OF 1.2, this function will also handle sending out the
-     * OF 1.2-style role request message.
-     *
-     * @param switches the collection of switches to send the request too
-     * @param role     the role to request
-     */
-    @LogMessageDoc(level = "WARN",
-            message = "Failed to send role request message " +
-                    "to switch {switch}: {message}. Disconnecting",
-            explanation = "An I/O error occurred while attempting to change " +
-                    "the switch HA role.",
-            recommendation = LogMessageDoc.CHECK_SWITCH)
-    protected void sendRoleRequest(Collection<OFSwitchImpl> switches,
-                                   Role role, long cookie) {
-        // There are three cases to consider:
-        //
-        // 1) If the controller role at the point the switch connected was
-        //    null/disabled, then we never sent the role request probe to the
-        //    switch and therefore never set the SWITCH_SUPPORTS_NX_ROLE
-        //    attribute for the switch, so supportsNxRole is null. In that
-        //    case since we're now enabling role support for the controller
-        //    we should send out the role request probe/update to the switch.
-        //
-        // 2) If supportsNxRole == Boolean.TRUE then that means we've already
-        //    sent the role request probe to the switch and it replied with
-        //    a role reply message, so we know it supports role request
-        //    messages. Now we're changing the role and we want to send
-        //    it another role request message to inform it of the new role
-        //    for the controller.
-        //
-        // 3) If supportsNxRole == Boolean.FALSE, then that means we sent the
-        //    role request probe to the switch but it responded with an error
-        //    indicating that it didn't understand the role request message.
-        //    In that case we don't want to send it another role request that
-        //    it (still) doesn't understand. But if the new role of the
-        //    controller is SLAVE, then we don't want the switch to remain
-        //    connected to this controller. It might support the older serial
-        //    failover model for HA support, so we want to terminate the
-        //    connection and get it to initiate a connection with another
-        //    controller in its list of controllers. Eventually (hopefully, if
-        //    things are configured correctly) it will walk down its list of
-        //    controllers and connect to the current master controller.
-        Iterator<OFSwitchImpl> iter = switches.iterator();
-        while (iter.hasNext()) {
-            OFSwitchImpl sw = iter.next();
-            try {
-                Boolean supportsNxRole = (Boolean)
-                        sw.getAttribute(IOFSwitch.SWITCH_SUPPORTS_NX_ROLE);
-                if ((supportsNxRole == null) || supportsNxRole) {
-                    // Handle cases #1 and #2
-                    log.debug("Sending NxRoleRequest to {}", sw);
-                    sw.sendNxRoleRequest(role, cookie);
-                } else {
-                    if (role == Role.MASTER) {
-                        // ONOS extension:
-                        log.debug("Switch {} doesn't support NxRoleRequests, but sending " +
-                                "{} request anyway", sw, role);
-                        //Send the role request anyway, even though we know the switch
-                        //doesn't support it. The switch will give an error and in our
-                        //error handling code we will add the switch.
-                        //NOTE we *could* just add the switch right away rather than
-                        //going through the overhead of sending a role request - however
-                        //we then have to deal with concurrency issues resulting from
-                        //calling addSwitch outside of a netty handler.
-                        sw.sendNxRoleRequest(role, cookie);
-                    }
-                    // Handle case #3
-                    else if (role == Role.SLAVE) {
-                        log.debug("Disconnecting switch {} that doesn't support " +
-                                "role request messages from a controller that went to SLAVE mode");
-                        // Closing the channel should result in a call to
-                        // channelDisconnect which updates all state 
-                        sw.getChannel().close();
-                        iter.remove();
-                    }
-                }
-            } catch (IOException e) {
-                log.warn("Failed to send role request message " +
-                        "to switch {}: {}. Disconnecting",
-                        sw, e);
-                sw.getChannel().close();
-                iter.remove();
-            }
-        }
-    }
-
-    /**
-     * Verify that switches have received a role reply message we sent earlier
-     *
-     * @param switches the collection of switches to send the request too
-     * @param cookie   the cookie of the request
-     */
-    @LogMessageDoc(level = "WARN",
-            message = "Timeout while waiting for role reply from switch {switch}."
-                    + " Disconnecting",
-            explanation = "Timed out waiting for the switch to respond to " +
-                    "a request to change the HA role.",
-            recommendation = LogMessageDoc.CHECK_SWITCH)
-    protected void verifyRoleReplyReceived(Collection<OFSwitchImpl> switches,
-                                           long cookie) {
-        for (OFSwitchImpl sw : switches) {
-            if (sw.checkFirstPendingRoleRequestCookie(cookie)) {
-                sw.getChannel().close();
-                log.warn("Timeout while waiting for role reply from switch {}."
-                        + " Disconnecting", sw);
-            }
-        }
-    }
-}
diff --git a/src/main/java/net/floodlightcontroller/core/internal/SwitchStateException.java b/src/main/java/net/floodlightcontroller/core/internal/SwitchStateException.java
index 8e49799..164346b 100644
--- a/src/main/java/net/floodlightcontroller/core/internal/SwitchStateException.java
+++ b/src/main/java/net/floodlightcontroller/core/internal/SwitchStateException.java
@@ -18,6 +18,13 @@
 package net.floodlightcontroller.core.internal;
 
 /**
+ * This exception indicates an error or unexpected message during
+ * message handling. E.g., if an OFMessage is received that is illegal or
+ * unexpected given the current handshake state.
+ *
+ * We don't allow wrapping other exception in a switch state exception. We
+ * only log the SwitchStateExceptions message so the causing exceptions
+ * stack trace is generally not available.
  *
  */
 public class SwitchStateException extends Exception {
diff --git a/src/main/java/net/floodlightcontroller/core/module/FloodlightModuleLoader.java b/src/main/java/net/floodlightcontroller/core/module/FloodlightModuleLoader.java
index 6913b1c..35ccc08 100644
--- a/src/main/java/net/floodlightcontroller/core/module/FloodlightModuleLoader.java
+++ b/src/main/java/net/floodlightcontroller/core/module/FloodlightModuleLoader.java
@@ -442,8 +442,10 @@
      * Call each loaded module's startup method
      *
      * @param moduleSet the module set to start up
+     * @throws FloodlightModuleException
      */
-    protected void startupModules(Collection<IFloodlightModule> moduleSet) {
+    protected void startupModules(Collection<IFloodlightModule> moduleSet)
+		throws FloodlightModuleException {
         for (IFloodlightModule m : moduleSet) {
             if (logger.isDebugEnabled()) {
                 logger.debug("Starting " + m.getClass().getCanonicalName());
diff --git a/src/main/java/net/floodlightcontroller/core/module/IFloodlightModule.java b/src/main/java/net/floodlightcontroller/core/module/IFloodlightModule.java
index a0ee845..ec08db1 100644
--- a/src/main/java/net/floodlightcontroller/core/module/IFloodlightModule.java
+++ b/src/main/java/net/floodlightcontroller/core/module/IFloodlightModule.java
@@ -73,5 +73,5 @@
      * @param context
      */
 
-    void startUp(FloodlightModuleContext context);
+    void startUp(FloodlightModuleContext context)throws FloodlightModuleException;
 }
diff --git a/src/main/java/net/floodlightcontroller/core/web/SwitchResourceBase.java b/src/main/java/net/floodlightcontroller/core/web/SwitchResourceBase.java
index 47f6175..d5b534d 100644
--- a/src/main/java/net/floodlightcontroller/core/web/SwitchResourceBase.java
+++ b/src/main/java/net/floodlightcontroller/core/web/SwitchResourceBase.java
@@ -117,12 +117,13 @@
                 // pass - nothing todo besides set the type above
             }
             req.setLengthU(requestLength);
-            try {
+            // XXX S fix when we fix stats
+            /*try {
                 future = sw.getStatistics(req);
                 values = future.get(10, TimeUnit.SECONDS);
             } catch (Exception e) {
                 log.error("Failure retrieving statistics from switch " + sw, e);
-            }
+            }*/
         }
         return values;
     }
@@ -140,12 +141,13 @@
         Future<OFFeaturesReply> future;
         OFFeaturesReply featuresReply = null;
         if (sw != null) {
-            try {
+		// XXX S fix when we fix stats
+            /*try {
                 future = sw.getFeaturesReplyFromSwitch();
                 featuresReply = future.get(10, TimeUnit.SECONDS);
             } catch (Exception e) {
                 log.error("Failure getting features reply from switch" + sw, e);
-            }
+            }*/
         }
 
         return featuresReply;
diff --git a/src/main/java/net/floodlightcontroller/debugcounter/DebugCounter.java b/src/main/java/net/floodlightcontroller/debugcounter/DebugCounter.java
new file mode 100644
index 0000000..b7c17b2
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugcounter/DebugCounter.java
@@ -0,0 +1,741 @@
+package net.floodlightcontroller.debugcounter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
+import net.floodlightcontroller.core.module.IFloodlightModule;
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.debugcounter.web.DebugCounterRoutable;
+import net.floodlightcontroller.restserver.IRestApiService;
+
+/**
+ * This class implements a central store for all counters used for debugging the
+ * system. For counters based on traffic-type, see ICounterStoreService.
+ *
+ * @author saurav
+ */
+public class DebugCounter implements IFloodlightModule, IDebugCounterService {
+    protected static Logger log = LoggerFactory.getLogger(DebugCounter.class);
+
+    /**
+     * registered counters need a counter id
+     */
+    protected AtomicInteger counterIdCounter = new AtomicInteger();
+
+    /**
+     * The counter value
+     */
+    protected class MutableLong {
+        long value = 0;
+        public void increment() { value += 1; }
+        public void increment(long incr) { value += incr; }
+        public long get() { return value; }
+        public void set(long val) { value = val; }
+      }
+
+    /**
+     * protected class to store counter information
+     */
+    public static class CounterInfo {
+        String moduleCounterHierarchy;
+        String counterDesc;
+        CounterType ctype;
+        String moduleName;
+        String counterHierarchy;
+        int counterId;
+        boolean enabled;
+        String[] metaData;
+
+        public CounterInfo(int counterId, boolean enabled,
+                           String moduleName, String counterHierarchy,
+                           String desc, CounterType ctype, String... metaData) {
+            this.moduleCounterHierarchy = moduleName + "/" + counterHierarchy;
+            this.moduleName = moduleName;
+            this.counterHierarchy = counterHierarchy;
+            this.counterDesc = desc;
+            this.ctype = ctype;
+            this.counterId = counterId;
+            this.enabled = enabled;
+            this.metaData = metaData;
+        }
+
+        public String getModuleCounterHierarchy() { return moduleCounterHierarchy; }
+        public String getCounterDesc() { return counterDesc; }
+        public CounterType getCtype() { return ctype; }
+        public String getModuleName() { return moduleName; }
+        public String getCounterHierarchy() { return counterHierarchy; }
+        public int getCounterId() { return counterId; }
+        public boolean isEnabled() { return enabled; }
+        public String[] getMetaData() { return metaData; }
+    }
+
+    //******************
+    //   Global stores
+    //******************
+
+    /**
+     * Counter info for a debug counter
+     */
+    public class DebugCounterInfo {
+        CounterInfo cinfo;
+        AtomicLong cvalue;
+
+        public DebugCounterInfo(CounterInfo cinfo) {
+            this.cinfo = cinfo;
+            this.cvalue = new AtomicLong();
+        }
+        public CounterInfo getCounterInfo() {
+            return cinfo;
+        }
+        public Long getCounterValue() {
+            return cvalue.get();
+        }
+    }
+
+    /**
+     * Global debug-counter storage across all threads. These are
+     * updated from the local per thread counters by the flush counters method.
+     */
+    protected static DebugCounterInfo[] allCounters =
+                            new DebugCounterInfo[MAX_COUNTERS];
+
+
+    /**
+     * per module counters, indexed by the module name and storing three levels
+     * of Counter information in the form of CounterIndexStore
+     */
+    protected ConcurrentHashMap<String, ConcurrentHashMap<String, CounterIndexStore>>
+        moduleCounters = new ConcurrentHashMap<String,
+                                                ConcurrentHashMap<String,
+                                                                   CounterIndexStore>>();
+
+    protected class CounterIndexStore {
+        int index;
+        Map<String, CounterIndexStore> nextLevel;
+
+        public CounterIndexStore(int index, Map<String,CounterIndexStore> cis) {
+            this.index = index;
+            this.nextLevel = cis;
+        }
+    }
+
+    /**
+     * fast global cache for counter ids that are currently active
+     */
+    protected Set<Integer> currentCounters = Collections.newSetFromMap(
+                                         new ConcurrentHashMap<Integer,Boolean>());
+
+    //******************
+    // Thread local stores
+    //******************
+
+    /**
+     * Thread local storage of counter info
+     */
+    protected class LocalCounterInfo {
+        boolean enabled;
+        MutableLong cvalue;
+
+        public LocalCounterInfo(boolean enabled) {
+            this.enabled = enabled;
+            this.cvalue = new MutableLong();
+        }
+    }
+
+    /**
+     * Thread local debug counters used for maintaining counters local to a thread.
+     */
+    protected final ThreadLocal<LocalCounterInfo[]> threadlocalCounters =
+            new ThreadLocal<LocalCounterInfo[]>() {
+        @Override
+        protected LocalCounterInfo[] initialValue() {
+            return new LocalCounterInfo[MAX_COUNTERS];
+        }
+    };
+
+    /**
+     * Thread local cache for counter ids that are currently active.
+     */
+    protected final ThreadLocal<Set<Integer>> threadlocalCurrentCounters =
+            new ThreadLocal<Set<Integer>>() {
+        @Override
+        protected Set<Integer> initialValue() {
+            return new HashSet<Integer>();
+        }
+    };
+
+    //*******************************
+    //   IDebugCounter
+    //*******************************
+
+    protected class CounterImpl implements IDebugCounter {
+        private final int counterId;
+
+        public CounterImpl(int counterId) {
+            this.counterId = counterId;
+        }
+
+        @Override
+        public void updateCounterWithFlush() {
+            if (!validCounterId()) return;
+            updateCounter(counterId, 1, true);
+        }
+
+        @Override
+        public void updateCounterNoFlush() {
+            if (!validCounterId()) return;
+            updateCounter(counterId, 1, false);
+        }
+
+        @Override
+        public void updateCounterWithFlush(int incr) {
+            if (!validCounterId()) return;
+            updateCounter(counterId, incr, true);
+        }
+
+        @Override
+        public void updateCounterNoFlush(int incr) {
+            if (!validCounterId()) return;
+            updateCounter(counterId, incr, false);
+        }
+
+        @Override
+        public long getCounterValue() {
+            if (!validCounterId()) return -1;
+            return allCounters[counterId].cvalue.get();
+        }
+
+        private boolean validCounterId() {
+            if (counterId < 0 || counterId >= MAX_COUNTERS) {
+                log.error("Invalid counterId invoked");
+                return false;
+            }
+            return true;
+        }
+
+    }
+
+   //*******************************
+   //   IDebugCounterService
+   //*******************************
+
+   @Override
+   public IDebugCounter registerCounter(String moduleName, String counterHierarchy,
+                           String counterDescription, CounterType counterType,
+                           String... metaData)
+               throws MaxCountersRegistered, MaxHierarchyRegistered,
+                      MissingHierarchicalLevel {
+       // check if counter already exists
+       if (!moduleCounters.containsKey(moduleName)) {
+           moduleCounters.putIfAbsent(moduleName,
+                new ConcurrentHashMap<String, CounterIndexStore>());
+       }
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (rci.allLevelsFound) {
+           // counter exists
+           log.info("Counter exists for {}/{} -- resetting counters", moduleName,
+                    counterHierarchy);
+           resetCounterHierarchy(moduleName, counterHierarchy);
+           return new CounterImpl(rci.ctrIds[rci.foundUptoLevel-1]);
+       }
+       // check for validity of counter
+       if (rci.levels.length > MAX_HIERARCHY) {
+           String err = "Registry of counterHierarchy " + counterHierarchy +
+                   " exceeds max hierachy " + MAX_HIERARCHY + ".. aborting";
+           throw new MaxHierarchyRegistered(err);
+       }
+       if (rci.foundUptoLevel < rci.levels.length-1) {
+           String needToRegister = "";
+           for (int i=0; i<=rci.foundUptoLevel; i++) {
+               needToRegister += rci.levels[i];
+           }
+           String err = "Attempting to register hierarchical counterHierarchy " +
+                   counterHierarchy + " but parts of hierarchy missing. " +
+                   "Please register " +  needToRegister + " first";
+           throw new MissingHierarchicalLevel(err);
+       }
+
+       // get a new counter id
+       int counterId = counterIdCounter.getAndIncrement();
+       if (counterId >= MAX_COUNTERS) {
+           throw new MaxCountersRegistered("max counters reached");
+       }
+       // create storage for counter
+       boolean enabled = (counterType == CounterType.ALWAYS_COUNT) ? true : false;
+       CounterInfo ci = new CounterInfo(counterId, enabled, moduleName,
+                                        counterHierarchy, counterDescription,
+                                        counterType, metaData);
+       allCounters[counterId] = new DebugCounterInfo(ci);
+
+       // account for the new counter in the module counter hierarchy
+       addToModuleCounterHierarchy(moduleName, counterId, rci);
+
+       // finally add to active counters
+       if (enabled) {
+           currentCounters.add(counterId);
+       }
+       return new CounterImpl(counterId);
+   }
+
+   private void updateCounter(int counterId, int incr, boolean flushNow) {
+       if (counterId < 0 || counterId >= MAX_COUNTERS) return;
+
+       LocalCounterInfo[] thiscounters =  this.threadlocalCounters.get();
+       if (thiscounters[counterId] == null) {
+           // seeing this counter for the first time in this thread - create local
+           // store by consulting global store
+           DebugCounterInfo dc = allCounters[counterId];
+           if (dc != null) {
+               thiscounters[counterId] = new LocalCounterInfo(dc.cinfo.enabled);
+               if (dc.cinfo.enabled) {
+                   Set<Integer> thisset = this.threadlocalCurrentCounters.get();
+                   thisset.add(counterId);
+               }
+           } else {
+               log.error("updateCounter seen locally for counter {} but no global"
+                          + "storage exists for it yet .. not updating", counterId);
+               return;
+           }
+       }
+
+       // update local store if enabled locally for updating
+       LocalCounterInfo lc = thiscounters[counterId];
+       if (lc.enabled) {
+           lc.cvalue.increment(incr);
+           if (flushNow) {
+               DebugCounterInfo dc = allCounters[counterId];
+               if (dc.cinfo.enabled) {
+                   // globally enabled - flush now
+                   dc.cvalue.addAndGet(lc.cvalue.get());
+                   lc.cvalue.set(0);
+               } else {
+                   // global counter is disabled - don't flush, disable locally
+                   lc.enabled = false;
+                   Set<Integer> thisset = this.threadlocalCurrentCounters.get();
+                   thisset.remove(counterId);
+               }
+           }
+       }
+   }
+
+   @Override
+   public void flushCounters() {
+       LocalCounterInfo[] thiscounters =  this.threadlocalCounters.get();
+       Set<Integer> thisset = this.threadlocalCurrentCounters.get();
+       ArrayList<Integer> temp = new ArrayList<Integer>();
+
+       for (int counterId : thisset) {
+           LocalCounterInfo lc = thiscounters[counterId];
+           if (lc.cvalue.get() > 0) {
+               DebugCounterInfo dc = allCounters[counterId];
+               if (dc.cinfo.enabled) {
+                   // globally enabled - flush now
+                   dc.cvalue.addAndGet(lc.cvalue.get());
+                   lc.cvalue.set(0);
+               } else {
+                   // global counter is disabled - don't flush, disable locally
+                   lc.enabled = false;
+                   temp.add(counterId);
+               }
+           }
+       }
+       for (int cId : temp) {
+           thisset.remove(cId);
+       }
+
+       // At this point it is possible that the thread-local set does not
+       // include a counter that has been enabled and is present in the global set.
+       // We need to sync thread-local currently enabled set of counterIds with
+       // the global set.
+       Sets.SetView<Integer> sv = Sets.difference(currentCounters, thisset);
+       for (int counterId : sv) {
+           if (thiscounters[counterId] != null) {
+               thiscounters[counterId].enabled = true;
+               thisset.add(counterId);
+           }
+       }
+   }
+
+   @Override
+   public void resetCounterHierarchy(String moduleName, String counterHierarchy) {
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (!rci.allLevelsFound) {
+           String missing = rci.levels[rci.foundUptoLevel];
+           log.error("Cannot reset counter hierarchy - missing counter {}", missing);
+           return;
+       }
+       // reset at this level
+       allCounters[rci.ctrIds[rci.foundUptoLevel-1]].cvalue.set(0);
+       // reset all levels below
+       ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+       for (int index : resetIds) {
+           allCounters[index].cvalue.set(0);
+       }
+   }
+
+   @Override
+   public void resetAllCounters() {
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+       for (String moduleName : moduleCounters.keySet()) {
+           ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+           for (int index : resetIds) {
+               allCounters[index].cvalue.set(0);
+           }
+       }
+   }
+
+   @Override
+   public void resetAllModuleCounters(String moduleName) {
+       Map<String, CounterIndexStore> target = moduleCounters.get(moduleName);
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+
+       if (target != null) {
+           ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+           for (int index : resetIds) {
+               allCounters[index].cvalue.set(0);
+           }
+       } else {
+           if (log.isDebugEnabled())
+               log.debug("No module found with name {}", moduleName);
+       }
+   }
+
+   @Override
+   public void enableCtrOnDemand(String moduleName, String counterHierarchy) {
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (!rci.allLevelsFound) {
+           String missing = rci.levels[rci.foundUptoLevel];
+           log.error("Cannot enable counter - counter not found {}", missing);
+           return;
+       }
+       // enable specific counter
+       DebugCounterInfo dc = allCounters[rci.ctrIds[rci.foundUptoLevel-1]];
+       dc.cinfo.enabled = true;
+       currentCounters.add(dc.cinfo.counterId);
+   }
+
+   @Override
+   public void disableCtrOnDemand(String moduleName, String counterHierarchy) {
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (!rci.allLevelsFound) {
+           String missing = rci.levels[rci.foundUptoLevel];
+           log.error("Cannot disable counter - counter not found {}", missing);
+           return;
+       }
+       // disable specific counter
+       DebugCounterInfo dc = allCounters[rci.ctrIds[rci.foundUptoLevel-1]];
+       if (dc.cinfo.ctype == CounterType.COUNT_ON_DEMAND) {
+           dc.cinfo.enabled = false;
+           dc.cvalue.set(0);
+           currentCounters.remove(dc.cinfo.counterId);
+       }
+   }
+
+   @Override
+   public List<DebugCounterInfo> getCounterHierarchy(String moduleName,
+                                                     String counterHierarchy) {
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       if (!rci.allLevelsFound) {
+           String missing = rci.levels[rci.foundUptoLevel];
+           log.error("Cannot fetch counter - counter not found {}", missing);
+           return Collections.emptyList();
+       }
+       ArrayList<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>();
+       // get counter and all below it
+       DebugCounterInfo dc = allCounters[rci.ctrIds[rci.foundUptoLevel-1]];
+       dcilist.add(dc);
+       ArrayList<Integer> belowIds = getHierarchyBelow(moduleName, rci);
+       for (int index : belowIds) {
+           dcilist.add(allCounters[index]);
+       }
+       return dcilist;
+   }
+
+   @Override
+   public List<DebugCounterInfo> getAllCounterValues() {
+       List<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>();
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+
+       for (String moduleName : moduleCounters.keySet()) {
+           ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+           for (int index : resetIds) {
+               dcilist.add(allCounters[index]);
+           }
+       }
+       return dcilist;
+   }
+
+   @Override
+   public List<DebugCounterInfo> getModuleCounterValues(String moduleName) {
+       List<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>();
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+
+       if (moduleCounters.containsKey(moduleName)) {
+           ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci);
+           for (int index : resetIds) {
+               dcilist.add(allCounters[index]);
+           }
+       }
+       return dcilist;
+   }
+
+   @Override
+   public boolean containsModuleCounterHierarchy(String moduleName,
+                                                 String counterHierarchy) {
+       if (!moduleCounters.containsKey(moduleName)) return false;
+       RetCtrInfo rci = getCounterId(moduleName, counterHierarchy);
+       return rci.allLevelsFound;
+   }
+
+   @Override
+   public boolean containsModuleName(String moduleName) {
+       return  (moduleCounters.containsKey(moduleName)) ? true : false;
+   }
+
+   @Override
+   public List<String> getModuleList() {
+       List<String> retval = new ArrayList<String>();
+       retval.addAll(moduleCounters.keySet());
+       return retval;
+   }
+
+   @Override
+   public List<String> getModuleCounterList(String moduleName) {
+       if (!moduleCounters.containsKey(moduleName))
+           return Collections.emptyList();
+
+       List<String> retval = new ArrayList<String>();
+       RetCtrInfo rci = new RetCtrInfo();
+       rci.levels = "".split("/");
+
+       ArrayList<Integer> cids = getHierarchyBelow(moduleName, rci);
+       for (int index : cids) {
+           retval.add(allCounters[index].cinfo.counterHierarchy);
+       }
+       return retval;
+   }
+
+   //*******************************
+   //   Internal Methods
+   //*******************************
+
+   protected class RetCtrInfo {
+       boolean allLevelsFound; // counter indices found all the way down the hierarchy
+       boolean hierarchical; // true if counterHierarchy is hierarchical
+       int foundUptoLevel;
+       int[]  ctrIds;
+       String[] levels;
+
+       public RetCtrInfo() {
+           ctrIds = new int[MAX_HIERARCHY];
+           for (int i=0; i<MAX_HIERARCHY; i++) {
+               ctrIds[i] = -1;
+           }
+       }
+
+       @Override
+       public boolean equals(Object oth) {
+           if (!(oth instanceof RetCtrInfo)) return false;
+           RetCtrInfo other = (RetCtrInfo)oth;
+           if (other.allLevelsFound != this.allLevelsFound) return false;
+           if (other.hierarchical != this.hierarchical) return false;
+           if (other.foundUptoLevel != this.foundUptoLevel) return false;
+           if (!Arrays.equals(other.ctrIds, this.ctrIds)) return false;
+           if (!Arrays.equals(other.levels, this.levels)) return false;
+           return true;
+       }
+
+   }
+
+   protected RetCtrInfo getCounterId(String moduleName, String counterHierarchy) {
+       RetCtrInfo rci = new RetCtrInfo();
+       Map<String, CounterIndexStore> templevel = moduleCounters.get(moduleName);
+       rci.levels = counterHierarchy.split("/");
+       if (rci.levels.length > 1) rci.hierarchical = true;
+       if (templevel == null) {
+           log.error("moduleName {} does not exist in debugCounters", moduleName);
+           return rci;
+       }
+
+       /*
+       if (rci.levels.length > MAX_HIERARCHY) {
+           // chop off all array elems greater that MAX_HIERARCHY
+           String[] temp = new String[MAX_HIERARCHY];
+           System.arraycopy(rci.levels, 0, temp, 0, MAX_HIERARCHY);
+           rci.levels = temp;
+       }
+       */
+       for (int i=0; i<rci.levels.length; i++) {
+           if (templevel != null) {
+               CounterIndexStore cis = templevel.get(rci.levels[i]) ;
+               if (cis == null) {
+                   // could not find counterHierarchy part at this level
+                   break;
+               } else {
+                   rci.ctrIds[i] = cis.index;
+                   templevel = cis.nextLevel;
+                   rci.foundUptoLevel++;
+                   if (i == rci.levels.length-1) {
+                       rci.allLevelsFound = true;
+                   }
+               }
+           } else {
+               // there are no more levels, which means that some part of the
+               // counterHierarchy has no corresponding map
+               break;
+           }
+       }
+       return rci;
+   }
+
+   protected void addToModuleCounterHierarchy(String moduleName, int counterId,
+                                            RetCtrInfo rci) {
+       Map<String, CounterIndexStore> target = moduleCounters.get(moduleName);
+       if (target == null) return;
+       CounterIndexStore cis = null;
+
+       for (int i=0; i<rci.foundUptoLevel; i++) {
+           cis = target.get(rci.levels[i]);
+           target = cis.nextLevel;
+       }
+       if (cis != null) {
+           if (cis.nextLevel == null)
+               cis.nextLevel = new ConcurrentHashMap<String, CounterIndexStore>();
+           cis.nextLevel.put(rci.levels[rci.foundUptoLevel],
+                             new CounterIndexStore(counterId, null));
+       } else {
+           target.put(rci.levels[rci.foundUptoLevel],
+                      new CounterIndexStore(counterId, null));
+       }
+   }
+
+   // given a partial hierarchical counter, return the rest of the hierarchy
+   protected ArrayList<Integer> getHierarchyBelow(String moduleName, RetCtrInfo rci) {
+       Map<String, CounterIndexStore> target = moduleCounters.get(moduleName);
+       CounterIndexStore cis = null;
+       ArrayList<Integer> retval = new ArrayList<Integer>();
+       if (target == null) return retval;
+
+       // get to the level given
+       for (int i=0; i<rci.foundUptoLevel; i++) {
+           cis = target.get(rci.levels[i]);
+           target = cis.nextLevel;
+       }
+
+       if (target == null || rci.foundUptoLevel == MAX_HIERARCHY) {
+           // no more levels
+           return retval;
+       } else {
+           // recursively get all ids
+           getIdsAtLevel(target, retval, rci.foundUptoLevel+1);
+       }
+
+       return retval;
+   }
+
+   protected void getIdsAtLevel(Map<String, CounterIndexStore> hcy,
+                                ArrayList<Integer> retval, int level) {
+       if (level > MAX_HIERARCHY) return;
+       if (hcy == null || retval == null) return;
+
+       // Can return the counter names as well but for now ids are enough.
+       for (CounterIndexStore cistemp : hcy.values()) {
+           retval.add(cistemp.index); // value at this level
+           if (cistemp.nextLevel != null) {
+               getIdsAtLevel(cistemp.nextLevel, retval, level+1);
+           }
+       }
+   }
+
+   protected void printAllCounterIds() {
+       log.info("<moduleCounterHierarchy>");
+       Set<String> keys = moduleCounters.keySet();
+       for (String key : keys) {
+           log.info("ModuleName: {}", key);
+           Map<String, CounterIndexStore> lev1 = moduleCounters.get(key);
+           for (String key1 : lev1.keySet()) {
+               CounterIndexStore cis1 = lev1.get(key1);
+               log.info(" L1 {}:{}", key1, new Object[] {cis1.index, cis1.nextLevel});
+               if (cis1.nextLevel != null) {
+                   Map<String, CounterIndexStore> lev2 = cis1.nextLevel;
+                   for (String key2 : lev2.keySet()) {
+                       CounterIndexStore cis2 = lev2.get(key2);
+                       log.info("  L2 {}:{}", key2, new Object[] {cis2.index,
+                                                                  cis2.nextLevel});
+                       if (cis2.nextLevel != null) {
+                           Map<String, CounterIndexStore> lev3 = cis2.nextLevel;
+                           for (String key3 : lev3.keySet()) {
+                               CounterIndexStore cis3 = lev3.get(key3);
+                               log.info("   L3 {}:{}", key3, new Object[] {cis3.index,
+                                                                          cis3.nextLevel});
+                           }
+                       }
+                   }
+               }
+           }
+       }
+       log.info("<\\moduleCounterHierarchy>");
+   }
+
+   //*******************************
+   //   IFloodlightModule
+   //*******************************
+
+   @Override
+   public Collection<Class<? extends IFloodlightService>> getModuleServices() {
+       Collection<Class<? extends IFloodlightService>> l =
+               new ArrayList<Class<? extends IFloodlightService>>();
+       l.add(IDebugCounterService.class);
+       return l;
+   }
+
+   @Override
+   public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() {
+       Map<Class<? extends IFloodlightService>, IFloodlightService> m =
+               new HashMap<Class<? extends IFloodlightService>, IFloodlightService>();
+       m.put(IDebugCounterService.class, this);
+       return m;
+   }
+
+   @Override
+   public Collection<Class<? extends IFloodlightService>> getModuleDependencies() {
+       ArrayList<Class<? extends IFloodlightService>> deps =
+               new ArrayList<Class<? extends IFloodlightService>>();
+       deps.add(IRestApiService.class);
+       return deps;
+   }
+
+   @Override
+   public void init(FloodlightModuleContext context) throws FloodlightModuleException {
+
+   }
+
+   @Override
+   public void startUp(FloodlightModuleContext context) {
+       IRestApiService restService =
+               context.getServiceImpl(IRestApiService.class);
+       restService.addRestletRoutable(new DebugCounterRoutable());
+   }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounter.java b/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounter.java
new file mode 100644
index 0000000..dbde185
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounter.java
@@ -0,0 +1,38 @@
+package net.floodlightcontroller.debugcounter;
+
+public interface IDebugCounter {
+    /**
+     * Increments the counter by 1 thread-locally, and immediately flushes to
+     * the global counter storage. This method should be used for counters that
+     * are updated outside the OF message processing pipeline.
+     */
+    void updateCounterWithFlush();
+
+    /**
+     * Increments the counter by 1 thread-locally. Flushing to the global
+     * counter storage is delayed (happens with flushCounters() in IDebugCounterService),
+     * resulting in higher performance. This method should be used for counters
+     * updated in the OF message processing pipeline.
+     */
+    void updateCounterNoFlush();
+
+    /**
+     * Increments the counter thread-locally by the 'incr' specified, and immediately
+     * flushes to the global counter storage. This method should be used for counters
+     * that are updated outside the OF message processing pipeline.
+     */
+    void updateCounterWithFlush(int incr);
+
+    /**
+     * Increments the counter thread-locally by the 'incr' specified. Flushing to the global
+     * counter storage is delayed (happens with flushCounters() in IDebugCounterService),
+     * resulting in higher performance. This method should be used for counters
+     * updated in the OF message processing pipeline.
+     */
+    void updateCounterNoFlush(int incr);
+
+    /**
+     * Retrieve the value of the counter from the global counter store
+     */
+    long getCounterValue();
+}
diff --git a/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounterService.java b/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounterService.java
new file mode 100644
index 0000000..f613e7b
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounterService.java
@@ -0,0 +1,260 @@
+package net.floodlightcontroller.debugcounter;
+
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.debugcounter.DebugCounter.DebugCounterInfo;
+
+import java.util.List;
+
+public interface IDebugCounterService extends IFloodlightService {
+
+    /**
+     * Different counter types. Counters that are meant to be counted-on-demand
+     * need to be separately enabled/disabled.
+     */
+    public enum CounterType {
+        ALWAYS_COUNT,
+        COUNT_ON_DEMAND
+    }
+
+    /**
+     * Debug Counter Qualifiers
+     */
+    public static final String CTR_MDATA_WARN = "warn";
+    public static final String CTR_MDATA_ERROR = "error";
+    public static final String CTR_MDATA_DROP = "drop";
+
+    /**
+     *  A limit on the maximum number of counters that can be created
+     */
+    public static final int MAX_COUNTERS = 5000;
+
+    /**
+     * Exception thrown when MAX_COUNTERS have been registered
+     */
+    public class MaxCountersRegistered extends CounterException {
+        private static final long serialVersionUID = 3173747663719376745L;
+        String errormsg;
+        public MaxCountersRegistered(String errormsg) {
+            this.errormsg = errormsg;
+        }
+        @Override
+        public String getMessage() {
+            return this.errormsg;
+        }
+    }
+    /**
+     * Exception thrown when MAX_HIERARCHY has been reached
+     */
+    public class MaxHierarchyRegistered extends CounterException {
+        private static final long serialVersionUID = 967431358683523871L;
+        String errormsg;
+        public MaxHierarchyRegistered(String errormsg) {
+            this.errormsg = errormsg;
+        }
+        @Override
+        public String getMessage() {
+            return this.errormsg;
+        }
+    }
+    /**
+     * Exception thrown when attempting to register a hierarchical counter
+     * where higher levels of the hierarchy have not been pre-registered
+     */
+    public class MissingHierarchicalLevel extends CounterException {
+        private static final long serialVersionUID = 517315311533995739L;
+        String errormsg;
+        public MissingHierarchicalLevel(String errormsg) {
+            this.errormsg = errormsg;
+        }
+        @Override
+        public String getMessage() {
+            return this.errormsg;
+        }
+    }
+
+    public class CounterException extends Exception {
+        private static final long serialVersionUID = 2219781500857866035L;
+    }
+
+    /**
+     *  maximum levels of hierarchy
+     *  Example of moduleName/counterHierarchy:
+     *           switch/00:00:00:00:01:02:03:04/pktin/drops where
+     *           moduleName ==> "switch"  and
+     *           counterHierarchy of 3 ==> "00:00:00:00:01:02:03:04/pktin/drops"
+     */
+    public static final int MAX_HIERARCHY = 3;
+
+    /**
+     * All modules that wish to have the DebugCounterService count for them, must
+     * register their counters by making this call (typically from that module's
+     * 'startUp' method). The counter can then be updated, displayed, reset etc.
+     * using the registered moduleName and counterHierarchy.
+     *
+     * @param moduleName           the name of the module which is registering the
+     *                             counter eg. linkdiscovery or controller or switch
+     * @param counterHierarchy     the hierarchical counter name specifying all
+     *                             the hierarchical levels that come above it.
+     *                             For example: to register a drop counter for
+     *                             packet-ins from a switch, the counterHierarchy
+     *                             can be "00:00:00:00:01:02:03:04/pktin/drops"
+     *                             It is necessary that counters in hierarchical levels
+     *                             above have already been pre-registered - in this
+     *                             example: "00:00:00:00:01:02:03:04/pktin" and
+     *                             "00:00:00:00:01:02:03:04"
+     * @param counterDescription   a descriptive string that gives more information
+     *                             of what the counter is measuring. For example,
+     *                             "Measures the number of incoming packets seen by
+     *                             this module".
+     * @param counterType          One of CounterType. On-demand counter types
+     *                             need to be explicitly enabled/disabled using other
+     *                             methods in this API -- i.e. registering them is
+     *                             not enough to start counting.
+     * @param metaData             variable arguments that qualify a counter
+     *                             eg. warn, error etc.
+     * @return                     IDebugCounter with update methods that can be
+     *                             used to update a counter.
+     * @throws MaxCountersRegistered
+     * @throws MaxHierarchyRegistered
+     * @throws MissingHierarchicalLevel
+     */
+    public IDebugCounter registerCounter(String moduleName, String counterHierarchy,
+                             String counterDescription, CounterType counterType,
+                             String... metaData)
+                throws MaxCountersRegistered, MaxHierarchyRegistered,
+                       MissingHierarchicalLevel;
+
+    /**
+     * Flush all thread-local counter values (from the current thread)
+     * to the global counter store. This method is not intended for use by any
+     * module. It's typical usage is from floodlight core and it is meant
+     * to flush those counters that are updated in the packet-processing pipeline,
+     * typically with the 'updateCounterNoFlush" methods in IDebugCounter.
+     */
+    public void flushCounters();
+
+    /**
+     * Resets the value of counters in the hierarchy to zero. Note that the reset
+     * applies to the level of counter hierarchy specified AND ALL LEVELS BELOW it
+     * in the hierarchy.
+     * For example: If a hierarchy exists like "00:00:00:00:01:02:03:04/pktin/drops"
+     *              specifying a reset hierarchy: "00:00:00:00:01:02:03:04"
+     *              will reset all counters for the switch dpid specified;
+     *              while specifying a reset hierarchy: ""00:00:00:00:01:02:03:04/pktin"
+     *              will reset the pktin counter and all levels below it (like drops)
+     *              for the switch dpid specified.
+     */
+    void resetCounterHierarchy(String moduleName, String counterHierarchy);
+
+    /**
+     * Resets the values of all counters in the system.
+     */
+    public void resetAllCounters();
+
+    /**
+     * Resets the values of all counters belonging
+     * to a module with the given 'moduleName'.
+     */
+    public void resetAllModuleCounters(String moduleName);
+
+    /**
+     * This method applies only to CounterType.COUNT_ON_DEMAND. It is used to
+     * enable counting on the counter. Note that this step is necessary to start
+     * counting for these counter types - merely registering the counter is not
+     * enough (as is the case for CounterType.ALWAYS_COUNT). Newly
+     * enabled counters start from an initial value of zero.
+     *
+     * Enabling a counter in a counterHierarchy enables only THAT counter. It
+     * does not enable any other part of the counterHierarchy. For example, if
+     * a hierarchy exists like "00:00:00:00:01:02:03:04/pktin/drops", where the
+     * 'pktin' and 'drops' counters are CounterType.COUNT_ON_DEMAND, then enabling
+     * the 'pktin' counter by specifying the counterHierarchy as
+     * "00:00:00:00:01:02:03:04/pktin" does NOT enable the 'drops' counter.
+     */
+    public void enableCtrOnDemand(String moduleName, String counterHierarchy);
+
+    /**
+     * This method applies only to CounterType.COUNT_ON_DEMAND. It is used to
+     * enable counting on the counter. Note that disabling a counter results in a loss
+     * of the counter value. When re-enabled the counter will restart from zero.
+     *
+     * Disabling a counter in a counterHierarchy disables only THAT counter. It
+     * does not disable any other part of the counterHierarchy. For example, if
+     * a hierarchy exists like "00:00:00:00:01:02:03:04/pktin/drops", where the
+     * 'pktin' and 'drops' counters are CounterType.COUNT_ON_DEMAND, then disabling
+     * the 'pktin' counter by specifying the counterHierarchy as
+     * "00:00:00:00:01:02:03:04/pktin" does NOT disable the 'drops' counter.
+     */
+    public void disableCtrOnDemand(String moduleName, String counterHierarchy);
+
+    /**
+     * Get counter value and associated information for the specified counterHierarchy.
+     * Note that information on the level of counter hierarchy specified
+     * AND ALL LEVELS BELOW it in the hierarchy will be returned.
+     *
+     * For example,
+     * if a hierarchy exists like "00:00:00:00:01:02:03:04/pktin/drops", then
+     * specifying a counterHierarchy of "00:00:00:00:01:02:03:04/pktin" in the
+     * get call will return information on the 'pktin' as well as the 'drops'
+     * counters for the switch dpid specified.
+     *
+     * @return A list of DebugCounterInfo or an empty list if the counter
+     *         could not be found
+     */
+    public List<DebugCounterInfo> getCounterHierarchy(String moduleName,
+                                                      String counterHierarchy);
+
+    /**
+     * Get counter values and associated information for all counters in the
+     * system
+     *
+     * @return the list of values/info or an empty list
+     */
+    public  List<DebugCounterInfo> getAllCounterValues();
+
+    /**
+     * Get counter values and associated information for all counters associated
+     * with a module.
+     *
+     * @param moduleName
+     * @return the list of values/info or an empty list
+     */
+    public  List<DebugCounterInfo> getModuleCounterValues(String moduleName);
+
+    /**
+     * Convenience method to figure out if the the given 'counterHierarchy' corresponds
+     * to a registered counterHierarchy for 'moduleName'. Note that the counter may or
+     * may not be enabled for counting, but if it is registered the method will
+     * return true.
+     *
+     * @param param
+     * @return false if moduleCounterHierarchy is not a registered counter
+     */
+    public boolean containsModuleCounterHierarchy(String moduleName,
+                                                  String counterHierarchy);
+
+    /**
+     * Convenience method to figure out if the the given 'moduleName' corresponds
+     * to a registered moduleName or not. Note that the module may or may not have
+     * a counter enabled for counting, but if it is registered the method will
+     * return true.
+     *
+     * @param param
+     * @return false if moduleName is not a registered counter
+     */
+    public boolean containsModuleName(String moduleName);
+
+    /**
+     * Returns a list of moduleNames registered for debug counters or an empty
+     * list if no counters have been registered in the system
+     */
+    public List<String> getModuleList();
+
+    /**
+     * Returns a list of all counters registered for a specific moduleName
+     * or a empty list
+     */
+    public List<String> getModuleCounterList(String moduleName);
+
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugcounter/NullDebugCounter.java b/src/main/java/net/floodlightcontroller/debugcounter/NullDebugCounter.java
new file mode 100644
index 0000000..e9081a8
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugcounter/NullDebugCounter.java
@@ -0,0 +1,163 @@
+package net.floodlightcontroller.debugcounter;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
+import net.floodlightcontroller.core.module.IFloodlightModule;
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.debugcounter.DebugCounter.DebugCounterInfo;
+
+public class NullDebugCounter implements IFloodlightModule, IDebugCounterService {
+
+
+    @Override
+    public Collection<Class<? extends IFloodlightService>>
+            getModuleServices() {
+        Collection<Class<? extends IFloodlightService>> services =
+                new ArrayList<Class<? extends IFloodlightService>>(1);
+        services.add(IDebugCounterService.class);
+        return services;
+    }
+
+    @Override
+    public Map<Class<? extends IFloodlightService>, IFloodlightService>
+            getServiceImpls() {
+        Map<Class<? extends IFloodlightService>,
+            IFloodlightService> m =
+                new HashMap<Class<? extends IFloodlightService>,
+                    IFloodlightService>();
+        m.put(IDebugCounterService.class, this);
+        return m;
+    }
+
+    @Override
+    public Collection<Class<? extends IFloodlightService>>
+            getModuleDependencies() {
+        return null;
+    }
+
+    @Override
+    public void init(FloodlightModuleContext context)
+            throws FloodlightModuleException {
+
+    }
+
+    @Override
+    public void startUp(FloodlightModuleContext context) {
+
+    }
+
+
+    @Override
+    public void flushCounters() {
+
+    }
+
+    @Override
+    public void resetAllCounters() {
+
+    }
+
+    @Override
+    public void resetAllModuleCounters(String moduleName) {
+
+    }
+
+
+    @Override
+    public void resetCounterHierarchy(String moduleName, String counterHierarchy) {
+
+    }
+
+    @Override
+    public void enableCtrOnDemand(String moduleName, String counterHierarchy) {
+
+    }
+
+    @Override
+    public void disableCtrOnDemand(String moduleName, String counterHierarchy) {
+
+    }
+
+    @Override
+    public List<DebugCounterInfo> getCounterHierarchy(String moduleName,
+                                                      String counterHierarchy) {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<DebugCounterInfo> getAllCounterValues() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<DebugCounterInfo> getModuleCounterValues(String moduleName) {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public boolean containsModuleCounterHierarchy(String moduleName,
+                                             String counterHierarchy) {
+        return false;
+    }
+
+    @Override
+    public boolean containsModuleName(String moduleName) {
+        return false;
+    }
+
+    @Override
+    public
+            IDebugCounter
+            registerCounter(String moduleName, String counterHierarchy,
+                            String counterDescription,
+                            CounterType counterType, String... metaData)
+                                 throws MaxCountersRegistered {
+        return new NullCounterImpl();
+    }
+
+    @Override
+    public List<String> getModuleList() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<String> getModuleCounterList(String moduleName) {
+        return Collections.emptyList();
+    }
+
+    public class NullCounterImpl implements IDebugCounter {
+
+        @Override
+        public void updateCounterWithFlush() {
+
+        }
+
+        @Override
+        public void updateCounterNoFlush() {
+
+        }
+
+        @Override
+        public void updateCounterWithFlush(int incr) {
+        }
+
+        @Override
+        public void updateCounterNoFlush(int incr) {
+
+        }
+
+        @Override
+        public long getCounterValue() {
+            return -1;
+        }
+
+    }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResource.java b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResource.java
new file mode 100644
index 0000000..3dd35bc
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResource.java
@@ -0,0 +1,382 @@
+package net.floodlightcontroller.debugcounter.web;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+import org.restlet.resource.Get;
+import org.restlet.resource.Post;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.floodlightcontroller.debugcounter.DebugCounter.DebugCounterInfo;
+import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType;
+
+/**
+ * Web interface for Debug Counters
+ *
+ * @author Saurav
+ */
+public class DebugCounterResource extends DebugCounterResourceBase {
+    protected static Logger logger =
+            LoggerFactory.getLogger(DebugCounterResource.class);
+
+    /**
+     * The output JSON model that contains the counter information
+     */
+    public class DebugCounterInfoOutput {
+        protected class DCInfo {
+            private final Long counterValue;
+            private final CounterType counterType;
+            private final String counterDesc;
+            private final boolean enabled;
+            private final String counterHierarchy;
+            private final String moduleName;
+            private final String[] metaData;
+
+            DCInfo(DebugCounterInfo dci) {
+                this.moduleName = dci.getCounterInfo().getModuleName();
+                this.counterHierarchy = dci.getCounterInfo().getCounterHierarchy();
+                this.counterDesc = dci.getCounterInfo().getCounterDesc();
+                this.metaData = dci.getCounterInfo().getMetaData();
+                this.enabled = dci.getCounterInfo().isEnabled();
+                this.counterType = dci.getCounterInfo().getCtype();
+                this.counterValue = dci.getCounterValue();
+            }
+
+            public Long getCounterValue() {
+                return counterValue;
+            }
+            public CounterType getCounterType() {
+                return counterType;
+            }
+
+            public String getCounterDesc() {
+                return counterDesc;
+            }
+
+            public boolean isEnabled() {
+                return enabled;
+            }
+
+            public String getCounterHierarchy() {
+                return counterHierarchy;
+            }
+
+            public String getModuleName() {
+                return moduleName;
+            }
+
+            public String[] getMetaData() {
+                return metaData;
+            }
+
+        }
+        // complete counter information - null if only names are requested or
+        // if an error occurs
+        public Map<String, DCInfo> counterMap;
+        // list of names could be just moduleNames or counter hierarchical names
+        // for a specific module
+        public List<String> names;
+
+        public String error;
+
+        DebugCounterInfoOutput(boolean getList) {
+            if (!getList) {
+                counterMap = new HashMap<String, DCInfo>();
+            }
+            error = null;
+        }
+        public Map<String, DCInfo> getCounterMap() {
+            return counterMap;
+        }
+
+        public String getError() {
+            return error;
+        }
+
+        public List<String> getNames() {
+            return names;
+        }
+
+    }
+
+    public enum Option {
+        ALL, ONE_MODULE, MODULE_COUNTER_HIERARCHY, ERROR_BAD_MODULE_NAME,
+        ERROR_BAD_PARAM,
+        ERROR_BAD_MODULE_COUNTER_NAME
+    }
+
+    public static class CounterPost {
+        public Boolean reset;
+        public Boolean enable;
+
+        public Boolean getReset() {
+            return reset;
+        }
+        public void setReset(Boolean reset) {
+            this.reset = reset;
+        }
+        public Boolean getEnable() {
+            return enable;
+        }
+        public void setEnable(Boolean enable) {
+            this.enable = enable;
+        }
+    }
+
+    public static class ResetOutput {
+        String error = null;
+
+        public String getError() {
+            return error;
+        }
+        public void setError(String error) {
+            this.error = error;
+        }
+    }
+
+    /**
+     * Reset or enable/disable counters
+     *
+     * If using curl:
+     * curl -X POST -d DATA -H "Content-Type: application/json" URL
+     * where  DATA must be one of the following:
+     *    {\"reset\":true}   to reset counters
+     *    {\"enable\":true}  to enable counter
+     *    {\"enable\":false} to disable counter
+     * and URL must be in one of the following forms:
+     *    "http://{controller-hostname}:8080/wm/debugcounter/{param1}/{param2}/{param3}/{param4}
+     *
+     * {param1} can be null, 'all' or the name of a module {moduleName}.
+     * {param2}/{param3}/{param4} refer to hierarchical counter names.
+     *
+     * The Reset command will reset the counter specified as well as all counters
+     * in the hierarchical levels below. For example, if a counter hierarchy exists
+     * as switch/00:00:00:00:01:02:03:04/pktin/drops, then a reset command with just
+     * the moduleName (param1=switch) and counterHierarchy (param2=00:00:00:00:01:02:03:04)
+     * will reset all counters for that switch. Continuing the example -
+     * for a counterHierarchy (param2=00:00:00:00:01:02:03:04 and param3=pktin), the reset
+     * command will remove all pktin counters for that switch.
+     *
+     * The enable/disable command will ONLY disable a specific counter (and only if
+     * that counter is of CounterType.ON_DEMAND)
+     * It will not enable/disable counters at any other hierarchical level.
+     *
+     */
+    @Post
+    public ResetOutput postHandler(CounterPost postData) {
+        ResetOutput output = new ResetOutput();
+        Option choice = Option.ERROR_BAD_PARAM;
+        String param1 = (String)getRequestAttributes().get("param1");
+        String param2 = (String)getRequestAttributes().get("param2");
+        String param3 = (String)getRequestAttributes().get("param3");
+        String param4 = (String)getRequestAttributes().get("param4");
+        String moduleName = "";
+
+        if (param1 == null) {
+             moduleName = "all";
+            choice = Option.ALL;
+        } else if (param1.equals("all")) {
+            moduleName = "all";
+            choice = Option.ALL;
+        } else {
+            moduleName = param1;
+        }
+
+        String counterHierarchy = "";
+        if (param2 != null) {
+            counterHierarchy += param2;
+            if (param3 != null) {
+                counterHierarchy += "/" + param3;
+                if (param4 != null) {
+                    counterHierarchy += "/" + param4;
+                }
+            }
+        }
+
+        if (!moduleName.equals("all") && counterHierarchy.equals("")) {
+            // only module name specified
+            boolean isRegistered = debugCounter.containsModuleName(param1);
+            if (isRegistered) {
+                choice = Option.ONE_MODULE;
+            } else {
+                choice = Option.ERROR_BAD_MODULE_NAME;
+            }
+        } else if (!moduleName.equals("all") && !counterHierarchy.equals("")) {
+            // both module and counter names specified
+            boolean isRegistered = debugCounter.
+                    containsModuleCounterHierarchy(moduleName, counterHierarchy);
+            if (isRegistered) {
+                choice = Option.MODULE_COUNTER_HIERARCHY;
+            } else {
+                choice = Option.ERROR_BAD_MODULE_COUNTER_NAME;
+            }
+        }
+
+        boolean reset = false;
+        boolean turnOnOff = false;
+        if (postData.getReset() != null && postData.getReset()) {
+            reset = true;
+        }
+        if (postData.getEnable() != null) {
+            turnOnOff = true;
+        }
+
+        switch (choice) {
+            case ALL:
+                if (reset) debugCounter.resetAllCounters();
+                break;
+            case ONE_MODULE:
+                if (reset) debugCounter.resetAllModuleCounters(moduleName);
+                break;
+            case MODULE_COUNTER_HIERARCHY:
+                if (reset)
+                    debugCounter.resetCounterHierarchy(moduleName, counterHierarchy);
+                else if (turnOnOff && postData.getEnable())
+                    debugCounter.enableCtrOnDemand(moduleName, counterHierarchy);
+                else if (turnOnOff && !postData.getEnable())
+                    debugCounter.disableCtrOnDemand(moduleName, counterHierarchy);
+                break;
+            case ERROR_BAD_MODULE_NAME:
+                output.error = "Module name has no corresponding registered counters";
+                break;
+            case ERROR_BAD_MODULE_COUNTER_NAME:
+                output.error = "Counter not registered";
+                break;
+            case ERROR_BAD_PARAM:
+                output.error = "Bad param";
+        }
+
+        return output;
+    }
+
+    /**
+     * Return the debug counter data for the get rest-api call
+     *
+     * URI must be in one of the following forms:
+     * "http://{controller-hostname}:8080/wm/debugcounter/{param1}/{param2}/{param3}/{param4}"
+     *
+     *  where {param1} must be one of (no quotes):
+     *       null                   if nothing is given then by default all
+     *                              module names are returned for which counters
+     *                              have been registered
+     *       "all"                  returns value/info on all counters.
+     *       "{moduleName}"         returns value/info on all counters for
+     *                              the specified module 'moduelName'.
+     * & {param2}/{param3}/{param4} refer to hierarchical counter names.
+     *                              eg. 00:00:00:00:01:02:03:04/pktin/drops
+     *                              where leaving out any of the params returns
+     *                              all counters in the hierarchical level below.
+     *                              So giving just the switch dpid will fetch
+     *                              all counters for that switch.
+     *                              A special case => if param2 is null, then
+     *                              all hierarchical counterNames are returned
+     *                              for the given moduleName (in param1)
+     */
+    @Get
+    public DebugCounterInfoOutput handleCounterInfoQuery() {
+        DebugCounterInfoOutput output;
+        Option choice = Option.ERROR_BAD_PARAM;
+        String param1 = (String)getRequestAttributes().get("param1");
+        String param2 = (String)getRequestAttributes().get("param2");
+        String param3 = (String)getRequestAttributes().get("param3");
+        String param4 = (String)getRequestAttributes().get("param4");
+
+        if (param1 == null) {
+            output = new DebugCounterInfoOutput(true);
+            return listCounters(output);
+        } else if (param1.equals("all")) {
+            output = new DebugCounterInfoOutput(false);
+            populateCounters(debugCounter.getAllCounterValues(), output);
+            return output;
+        }
+
+        output = new DebugCounterInfoOutput(false);
+        String counterHierarchy = "";
+        if (param2 == null) {
+            // param2 is null -- return list of counternames for param1
+            boolean isRegistered = debugCounter.containsModuleName(param1);
+            output = new DebugCounterInfoOutput(true);
+            if (isRegistered) {
+                return listCounters(param1, output);
+            } else {
+                choice = Option.ERROR_BAD_MODULE_NAME;
+            }
+        } else if (param2.equals("all")) {
+            // get all counter info for a single module
+            boolean isRegistered = debugCounter.containsModuleName(param1);
+            if (isRegistered) {
+                choice = Option.ONE_MODULE;
+            } else {
+                choice = Option.ERROR_BAD_MODULE_NAME;
+            }
+        } else {
+            counterHierarchy += param2;
+            if (param3 != null) {
+                counterHierarchy += "/" + param3;
+                if (param4 != null) {
+                    counterHierarchy += "/" + param4;
+                }
+            }
+            boolean isRegistered = debugCounter.
+                    containsModuleCounterHierarchy(param1, counterHierarchy);
+            if (isRegistered) {
+                choice = Option.MODULE_COUNTER_HIERARCHY;
+            } else {
+                choice = Option.ERROR_BAD_MODULE_COUNTER_NAME;
+            }
+        }
+
+        switch (choice) {
+            case ONE_MODULE:
+                populateCounters(debugCounter.getModuleCounterValues(param1), output);
+                break;
+            case MODULE_COUNTER_HIERARCHY:
+                populateCounters(debugCounter.getCounterHierarchy(param1, counterHierarchy),
+                                      output);
+                break;
+            case ERROR_BAD_MODULE_NAME:
+                output.error = "Module name is not registered for debug-counters";
+                break;
+            case ERROR_BAD_MODULE_COUNTER_NAME:
+                output.error = "Counter not registered";
+                break;
+            case ERROR_BAD_PARAM:
+            default:
+                output.error = "Bad param";
+        }
+
+        return output;
+    }
+
+    private DebugCounterInfoOutput listCounters(String moduleName,
+                                                DebugCounterInfoOutput output) {
+        output.names = debugCounter.getModuleCounterList(moduleName);
+        return output;
+    }
+
+    private DebugCounterInfoOutput listCounters(DebugCounterInfoOutput output) {
+        output.names = debugCounter.getModuleList();
+        return output;
+    }
+
+    private void populateSingleCounter(DebugCounterInfo debugCounterInfo,
+                                       DebugCounterInfoOutput output) {
+        if (debugCounterInfo != null)
+            output.counterMap.put(debugCounterInfo.getCounterInfo().
+                                  getModuleCounterHierarchy(),
+                                  output.new DCInfo(debugCounterInfo));
+    }
+
+    private void populateCounters(List<DebugCounterInfo> counterValues,
+                                        DebugCounterInfoOutput output) {
+        for (DebugCounterInfo dci : counterValues) {
+            populateSingleCounter(dci, output);
+        }
+    }
+
+
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResourceBase.java b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResourceBase.java
new file mode 100644
index 0000000..48e469b
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResourceBase.java
@@ -0,0 +1,18 @@
+package net.floodlightcontroller.debugcounter.web;
+
+import net.floodlightcontroller.debugcounter.IDebugCounterService;
+
+import org.restlet.resource.ResourceException;
+import org.restlet.resource.ServerResource;
+
+public class DebugCounterResourceBase extends ServerResource {
+
+    protected IDebugCounterService debugCounter;
+
+    @Override
+    protected void doInit() throws ResourceException {
+        super.doInit();
+        debugCounter = (IDebugCounterService)getContext().getAttributes().
+                get(IDebugCounterService.class.getCanonicalName());
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterRoutable.java b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterRoutable.java
new file mode 100644
index 0000000..55ba5ba
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterRoutable.java
@@ -0,0 +1,31 @@
+package net.floodlightcontroller.debugcounter.web;
+
+import org.restlet.Context;
+import org.restlet.Restlet;
+import org.restlet.routing.Router;
+
+import net.floodlightcontroller.restserver.RestletRoutable;
+
+public class DebugCounterRoutable implements RestletRoutable {
+
+    @Override
+    public String basePath() {
+        return "/wm/debugcounter";
+    }
+
+    @Override
+    public Restlet getRestlet(Context context) {
+        Router router = new Router(context);
+        router.attach("/{param1}/{param2}/{param3}/{param4}/", DebugCounterResource.class);
+        router.attach("/{param1}/{param2}/{param3}/{param4}", DebugCounterResource.class);
+        router.attach("/{param1}/{param2}/{param3}/", DebugCounterResource.class);
+        router.attach("/{param1}/{param2}/{param3}", DebugCounterResource.class);
+        router.attach("/{param1}/{param2}/", DebugCounterResource.class);
+        router.attach("/{param1}/{param2}", DebugCounterResource.class);
+        router.attach("/{param1}/", DebugCounterResource.class);
+        router.attach("/{param1}", DebugCounterResource.class);
+        router.attach("/", DebugCounterResource.class);
+        router.attach("", DebugCounterResource.class);
+        return router;
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/CircularBuffer.java b/src/main/java/net/floodlightcontroller/debugevent/CircularBuffer.java
new file mode 100644
index 0000000..f1a6db3
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/CircularBuffer.java
@@ -0,0 +1,95 @@
+package net.floodlightcontroller.debugevent;
+
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CircularBuffer<T> implements Iterable<T>{
+    protected static Logger log = LoggerFactory.getLogger(CircularBuffer.class);
+    private final LinkedBlockingDeque<T> buffer;
+
+    public CircularBuffer(int capacity) {
+        this.buffer = new LinkedBlockingDeque<T>(capacity);
+    }
+
+    /**
+     * Adding an element to the circular buffer implies adding the element to
+     * the tail of the deque. In doing so, if the capacity of the buffer has
+     * been exhausted, then the deque's head should be removed to preserve the
+     * circular nature of the buffer. A LinkedBlockingDeque is used because of its
+     * concurrent nature and the fact that adding to tail and removing from head are
+     * both O(1) operations. The removed head is returned to the caller for reuse.
+     *
+     * @param e    the element to be added
+     * @return     removed element (for reuse) or null
+     */
+    public T add(T e) {
+        T oldE = null;
+        while (!buffer.offerLast(e)) {
+            oldE = buffer.poll();
+        }
+        return oldE;
+    }
+
+    /**
+     * The basic idea here is that an ArrayList has been passed in, which may or may not
+     * have a size bigger that the actual number of elements that are meant to
+     * be flushed to the Circular Buffer. Thus the 'uptoIndex' parameter specifies
+     * the number of elements that need to be flushed starting from index 0.
+     * Note that after flushing, the circular buffer may return a memory unit (of type T)
+     * for reuse in the list, if the circular buffer popped off memory to preserve
+     * its circular nature. Or it may just return null if nothing was popped off.
+     * Either way, the list that is returned by this method, is of the SAME SIZE
+     * as the list passed in, as ArrayLists can hold null elements. The only difference
+     * is that the list returned will have elements that reference old popped-off memory
+     * from the circular-buffer or null.
+     *
+     * @param elist         the ArrayList to flush into the circular buffer.
+     * @param uptoIndex     flush starting from index 0 upto but not including
+     *                      index 'uptoIndex'.
+     * @return              the 'elist' passed in with members now pointing to
+     *                      to null or old-memory for reuse. The returned list
+     *                      if of the same size as 'elist'.
+     */
+    public ArrayList<T> addAll(ArrayList<T> elist, int uptoIndex) {
+        if (uptoIndex > elist.size()) {
+            log.error("uptoIndex is greater than elist size .. aborting addAll");
+            return elist;
+        }
+        for (int index=0; index < uptoIndex; index++) {
+            T e = elist.get(index);
+            if (e != null) {
+                elist.set(index, add(e));
+            }
+        }
+        return elist;
+    }
+
+    /**
+     * Returns an iterator over the elements in the circular buffer in proper sequence.
+     * The elements will be returned in order from most-recent to oldest.
+     * The returned Iterator is a "weakly consistent" iterator that will never
+     * throw ConcurrentModificationException, and guarantees to traverse elements
+     * as they existed upon construction of the iterator, and may (but is not
+     * guaranteed to) reflect any modifications subsequent to construction.
+     */
+    @Override
+    public Iterator<T> iterator() {
+        return buffer.descendingIterator();
+    }
+
+    public int size() {
+        return buffer.size();
+    }
+
+    /**
+     *  Atomically removes all elements in the circular buffer
+     */
+    public void clear() {
+        buffer.clear();
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/DebugEvent.java b/src/main/java/net/floodlightcontroller/debugevent/DebugEvent.java
new file mode 100644
index 0000000..e167b26
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/DebugEvent.java
@@ -0,0 +1,506 @@
+package net.floodlightcontroller.debugevent;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
+import net.floodlightcontroller.core.module.IFloodlightModule;
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.debugevent.web.DebugEventRoutable;
+import net.floodlightcontroller.restserver.IRestApiService;
+
+import com.google.common.collect.Sets;
+/**
+ * This class implements a central store for all events used for debugging the
+ * system. The basic idea is that given the functionality provided by this class,
+ * it should be unnecessary to resort to scraping through system DEBUG/TRACE logs
+ * to understand behavior in a running system.
+ *
+ * @author saurav
+ */
+public class DebugEvent implements IFloodlightModule, IDebugEventService {
+    protected static Logger log = LoggerFactory.getLogger(DebugEvent.class);
+
+    /**
+     *  Every registered event type gets an event id, the value for which is obtained
+     *  while holding the lock.
+     */
+    protected int eventIdCounter = 0;
+    protected Object eventIdLock = new Object();
+
+    private static final int PCT_LOCAL_CAP = 10; // % of global capacity
+    private static final int MIN_LOCAL_CAPACITY = 10; //elements
+
+    /**
+     * Event Information
+     */
+    public class EventInfo {
+        int eventId;
+        boolean enabled;
+        int bufferCapacity;
+        EventType etype;
+        String eventDesc;
+        String eventName;
+        String moduleName;
+        String moduleEventName;
+        Class<?> eventClass;
+        String[] metaData;
+
+        public EventInfo(int eventId, boolean enabled, int bufferCapacity,
+                         EventType etype, Class<?> eventClass, String eventDesc,
+                         String eventName, String moduleName, String... metaData) {
+            this.enabled = enabled;
+            this.eventId = eventId;
+            this.bufferCapacity = bufferCapacity;
+            this.etype = etype;
+            this.eventClass = eventClass;
+            this.eventDesc = eventDesc;
+            this.eventName = eventName;
+            this.moduleName = moduleName;
+            this.moduleEventName = moduleName + "/" + eventName;
+            this.metaData = metaData;
+        }
+
+        public int getEventId() { return eventId; }
+        public boolean isEnabled() { return enabled; }
+        public int getBufferCapacity() { return bufferCapacity; }
+        public EventType getEtype() { return etype; }
+        public String getEventDesc() { return eventDesc; }
+        public String getEventName() { return eventName; }
+        public String getModuleName() { return moduleName; }
+        public String getModuleEventName() { return moduleEventName; }
+        public String[] getMetaData() { return metaData; }
+    }
+
+    //******************
+    //   Global stores
+    //******************
+
+    /**
+     * Event history for a particular event-id is stored in a circular buffer
+     */
+    protected class DebugEventHistory {
+        EventInfo einfo;
+        CircularBuffer<Event> eventBuffer;
+
+        public DebugEventHistory(EventInfo einfo, int capacity) {
+            this.einfo = einfo;
+            this.eventBuffer = new CircularBuffer<Event>(capacity);
+        }
+    }
+
+    /**
+     * Global storage for all event types and their corresponding event buffers.
+     * A particular event type is accessed by directly indexing into the array
+     * with the corresponding event-id.
+     */
+    protected DebugEventHistory[] allEvents =
+                    new DebugEventHistory[MAX_EVENTS];
+
+    /**
+     * Global storage for all event ids registered for a module. The map is indexed
+     * by the module name and event name and returns the event-ids that correspond to the
+     * event types registered by that module (for example module 'linkdiscovery'
+     * may register events that have ids 0 and 1 that correspond to link up/down
+     * events, and receiving malformed LLDP packets, respectively).
+     */
+    protected ConcurrentHashMap<String, ConcurrentHashMap<String, Integer>>
+        moduleEvents = new ConcurrentHashMap<String,
+                                ConcurrentHashMap<String, Integer>>();
+
+    /**
+     * A collection of event ids that are currently enabled for logging
+     */
+    protected Set<Integer> currentEvents = Collections.newSetFromMap(
+                                       new ConcurrentHashMap<Integer,Boolean>());
+
+    //******************
+    // Thread local stores
+    //******************
+
+    /**
+     * Thread local storage for events
+     */
+    protected class LocalEventHistory {
+        int nextIndex;
+        int maxCapacity;
+        boolean enabled;
+        ArrayList<Event> eventList;
+
+        public LocalEventHistory(boolean enabled, int maxCapacity) {
+            this.nextIndex = 0;
+            this.maxCapacity = maxCapacity;
+            this.enabled = enabled;
+            this.eventList = new ArrayList<Event>();
+        }
+    }
+
+    /**
+     * Thread local event buffers used for maintaining event history local to
+     * a thread. Eventually this locally maintained information is flushed
+     * into the global event buffers.
+     */
+    protected final ThreadLocal<LocalEventHistory[]> threadlocalEvents =
+            new ThreadLocal<LocalEventHistory[]>() {
+        @Override
+        protected LocalEventHistory[] initialValue() {
+            return new LocalEventHistory[MAX_EVENTS];
+        }
+    };
+
+    /**
+     * Thread local cache for event-ids that are currently active.
+     */
+    protected final ThreadLocal<Set<Integer>> threadlocalCurrentEvents =
+            new ThreadLocal<Set<Integer>>() {
+        @Override
+        protected Set<Integer> initialValue() {
+            return new HashSet<Integer>();
+        }
+    };
+
+    //*******************************
+    //   IEventUpdater
+    //*******************************
+
+    protected class EventUpdaterImpl<T> implements IEventUpdater<T> {
+        private final int eventId;
+
+        public EventUpdaterImpl(int evId) {
+            this.eventId = evId;
+        }
+
+        @Override
+        public void updateEventNoFlush(Object event) {
+            if (!validEventId()) return;
+            updateEvent(eventId, false, event);
+        }
+
+        @Override
+        public void updateEventWithFlush(Object event) {
+            if (!validEventId()) return;
+            updateEvent(eventId, true, event);
+        }
+
+        private boolean validEventId() {
+            if (eventId < 0 || eventId >= MAX_EVENTS) {
+                throw new IllegalStateException();
+            }
+            return true;
+        }
+
+    }
+
+    //*******************************
+    //   IDebugEventService
+    //*******************************
+
+    @Override
+    public <T> IEventUpdater<T> registerEvent(String moduleName, String eventName,
+                                              String eventDescription, EventType et,
+                                              Class<T> eventClass, int bufferCapacity,
+                                              String... metaData) throws MaxEventsRegistered {
+        int eventId = -1;
+        synchronized (eventIdLock) {
+             eventId = Integer.valueOf(eventIdCounter++);
+        }
+        if (eventId > MAX_EVENTS-1) {
+            throw new MaxEventsRegistered();
+        }
+
+        // register event id for moduleName
+        if (!moduleEvents.containsKey(moduleName)) {
+            moduleEvents.put(moduleName, new ConcurrentHashMap<String, Integer>());
+        }
+        if (!moduleEvents.get(moduleName).containsKey(eventName)) {
+            moduleEvents.get(moduleName).put(eventName, new Integer(eventId));
+        } else {
+            int existingEventId = moduleEvents.get(moduleName).get(eventName);
+            log.error("Duplicate event registration for moduleName {} eventName {}",
+                      moduleName, eventName);
+            return new EventUpdaterImpl<T>(existingEventId);
+        }
+
+        // create storage for event-type
+        boolean enabled = (et == EventType.ALWAYS_LOG) ? true : false;
+        EventInfo ei = new EventInfo(eventId, enabled, bufferCapacity,
+                                     et, eventClass, eventDescription, eventName,
+                                     moduleName, metaData);
+        allEvents[eventId] = new DebugEventHistory(ei, bufferCapacity);
+        if (enabled) {
+            currentEvents.add(eventId);
+        }
+
+        return new EventUpdaterImpl<T>(eventId);
+    }
+
+    private void updateEvent(int eventId, boolean flushNow, Object eventData) {
+        if (eventId < 0 || eventId > MAX_EVENTS-1) return;
+
+        LocalEventHistory[] thishist = this.threadlocalEvents.get();
+        if (thishist[eventId] == null) {
+            // seeing this event for the first time in this thread - create local
+            // store by consulting global store
+            DebugEventHistory de = allEvents[eventId];
+            if (de != null) {
+                boolean enabled = de.einfo.enabled;
+                int localCapacity = de.einfo.bufferCapacity * PCT_LOCAL_CAP/ 100;
+                if (localCapacity < 10)  localCapacity = MIN_LOCAL_CAPACITY;
+                thishist[eventId] = new LocalEventHistory(enabled, localCapacity);
+                if (enabled) {
+                    Set<Integer> thisset = this.threadlocalCurrentEvents.get();
+                    thisset.add(eventId);
+                }
+            } else {
+                log.error("updateEvent seen locally for event {} but no global"
+                          + "storage exists for it yet .. not updating", eventId);
+                return;
+            }
+        }
+
+        // update local store if enabled locally for updating
+        LocalEventHistory le = thishist[eventId];
+        if (le.enabled) {
+            long timestamp = System.currentTimeMillis();
+            long thisthread = Thread.currentThread().getId();
+            String thisthreadname = Thread.currentThread().getName();
+            if (le.nextIndex < le.eventList.size()) {
+                if (le.eventList.get(le.nextIndex) == null) {
+                    le.eventList.set(le.nextIndex, new Event(timestamp, thisthread,
+                                                             thisthreadname,
+                                                             eventData));
+                } else {
+                    Event e = le.eventList.get(le.nextIndex);
+                    e.timestamp = timestamp;
+                    e.threadId = thisthread;
+                    e.threadName = thisthreadname;
+                    e.eventData = eventData;
+                    e.nullifyCachedFormattedEvent();
+                }
+            } else {
+                le.eventList.add(new Event(timestamp, thisthread, thisthreadname, eventData));
+            }
+            le.nextIndex++;
+
+            if (le.nextIndex >= le.maxCapacity || flushNow) {
+                // flush this buffer now
+                DebugEventHistory de = allEvents[eventId];
+                if (de.einfo.enabled) {
+                    le.eventList = de.eventBuffer.addAll(le.eventList, le.nextIndex);
+                } else {
+                    // global buffer is disabled - don't flush, disable locally
+                    le.enabled = false;
+                    Set<Integer> thisset = this.threadlocalCurrentEvents.get();
+                    thisset.remove(eventId);
+                }
+                le.nextIndex = 0;
+            }
+        }
+    }
+
+    @Override
+    public void flushEvents() {
+        LocalEventHistory[] thishist = this.threadlocalEvents.get();
+        Set<Integer> thisset = this.threadlocalCurrentEvents.get();
+        ArrayList<Integer> temp = new ArrayList<Integer>();
+
+        for (int eventId : thisset) {
+            LocalEventHistory le = thishist[eventId];
+            if (le != null && le.nextIndex > 0) {
+                // flush this buffer now
+                DebugEventHistory de = allEvents[eventId];
+                if (de.einfo.enabled) {
+                    le.eventList = de.eventBuffer.addAll(le.eventList, le.nextIndex);
+                } else {
+                    // global buffer is disabled - don't flush, disable locally
+                    le.enabled = false;
+                    temp.add(eventId);
+                }
+                le.nextIndex = 0;
+            }
+        }
+        for (int eId : temp)
+            thisset.remove(eId);
+
+        // sync thread local currently enabled set of eventIds with global set.
+        Sets.SetView<Integer> sv = Sets.difference(currentEvents, thisset);
+        for (int eventId : sv) {
+            if (thishist[eventId] != null) {
+                thishist[eventId].enabled = true;
+                thisset.add(eventId);
+            }
+        }
+
+    }
+
+    @Override
+    public boolean containsModuleEventName(String moduleName, String eventName) {
+        if (!moduleEvents.containsKey(moduleName)) return false;
+        if (moduleEvents.get(moduleName).containsKey(eventName)) return true;
+        return false;
+    }
+
+    @Override
+    public boolean containsModuleName(String moduleName) {
+        return moduleEvents.containsKey(moduleName);
+    }
+
+    @Override
+    public List<DebugEventInfo> getAllEventHistory() {
+        List<DebugEventInfo> moduleEventList = new ArrayList<DebugEventInfo>();
+        for (Map<String, Integer> modev : moduleEvents.values()) {
+            for (int eventId : modev.values()) {
+                DebugEventHistory de = allEvents[eventId];
+                if (de != null) {
+                    List<Map<String,String>> ret = new ArrayList<Map<String,String>>();
+                    for (Event e : de.eventBuffer) {
+                        ret.add(e.getFormattedEvent(de.einfo.eventClass,
+                                                    de.einfo.moduleEventName));
+                    }
+                    moduleEventList.add(new DebugEventInfo(de.einfo, ret));
+                }
+            }
+        }
+        return moduleEventList;
+    }
+
+    @Override
+    public List<DebugEventInfo> getModuleEventHistory(String moduleName) {
+        if (!moduleEvents.containsKey(moduleName)) return Collections.emptyList();
+        List<DebugEventInfo> moduleEventList = new ArrayList<DebugEventInfo>();
+        for (int eventId : moduleEvents.get(moduleName).values()) {
+            DebugEventHistory de = allEvents[eventId];
+            if (de != null) {
+                List<Map<String,String>> ret = new ArrayList<Map<String,String>>();
+                for (Event e : de.eventBuffer) {
+                    ret.add(e.getFormattedEvent(de.einfo.eventClass,
+                                                de.einfo.moduleEventName));
+                }
+                moduleEventList.add(new DebugEventInfo(de.einfo, ret));
+            }
+        }
+        return moduleEventList;
+    }
+
+    @Override
+    public DebugEventInfo getSingleEventHistory(String moduleName, String eventName,
+                                                int last) {
+        if (!moduleEvents.containsKey(moduleName)) return null;
+        Integer eventId = moduleEvents.get(moduleName).get(eventName);
+        if (eventId == null) return null;
+        DebugEventHistory de = allEvents[eventId];
+        if (de != null) {
+            int num = 1;
+            List<Map<String,String>> ret = new ArrayList<Map<String,String>>();
+            for (Event e : de.eventBuffer) {
+                if (num > last)
+                    break;
+                Map<String, String> temp = e.getFormattedEvent(de.einfo.eventClass,
+                                                               de.einfo.moduleEventName);
+                temp.put("#", String.valueOf(num++));
+                ret.add(temp);
+            }
+            return new DebugEventInfo(de.einfo, ret);
+        }
+        return null;
+    }
+
+    @Override
+    public void resetAllEvents() {
+        for (Map<String, Integer> eventMap : moduleEvents.values()) {
+            for (Integer evId : eventMap.values()) {
+                allEvents[evId].eventBuffer.clear();
+            }
+        }
+    }
+
+    @Override
+    public void resetAllModuleEvents(String moduleName) {
+        if (!moduleEvents.containsKey(moduleName)) return;
+        Map<String, Integer> modEvents = moduleEvents.get(moduleName);
+        for (Integer evId : modEvents.values()) {
+            allEvents[evId].eventBuffer.clear();
+        }
+    }
+
+    @Override
+    public void resetSingleEvent(String moduleName, String eventName) {
+        if (!moduleEvents.containsKey(moduleName)) return;
+        Integer eventId = moduleEvents.get(moduleName).get(eventName);
+        if (eventId == null) return;
+        DebugEventHistory de = allEvents[eventId];
+        if (de != null) {
+            de.eventBuffer.clear();
+        }
+    }
+
+    @Override
+    public List<String> getModuleList() {
+        List<String> el = new ArrayList<String>();
+        el.addAll(moduleEvents.keySet());
+        return el;
+    }
+
+    @Override
+    public List<String> getModuleEventList(String moduleName) {
+        if (!moduleEvents.containsKey(moduleName))
+            return Collections.emptyList();
+        List<String> el = new ArrayList<String>();
+        el.addAll(moduleEvents.get(moduleName).keySet());
+        return el;
+    }
+
+    //*******************************
+    //   IFloodlightModule
+    //*******************************
+
+    @Override
+    public Collection<Class<? extends IFloodlightService>> getModuleServices() {
+        Collection<Class<? extends IFloodlightService>> l =
+                new ArrayList<Class<? extends IFloodlightService>>();
+        l.add(IDebugEventService.class);
+        return l;
+    }
+
+    @Override
+    public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() {
+        Map<Class<? extends IFloodlightService>, IFloodlightService> m =
+                new HashMap<Class<? extends IFloodlightService>, IFloodlightService>();
+        m.put(IDebugEventService.class, this);
+        return m;
+    }
+
+    @Override
+    public Collection<Class<? extends IFloodlightService>> getModuleDependencies() {
+        ArrayList<Class<? extends IFloodlightService>> deps =
+                new ArrayList<Class<? extends IFloodlightService>>();
+        deps.add(IRestApiService.class);
+        return deps;
+    }
+
+    @Override
+    public void init(FloodlightModuleContext context)
+            throws FloodlightModuleException {
+    }
+
+    @Override
+    public void startUp(FloodlightModuleContext context)
+            throws FloodlightModuleException {
+        IRestApiService restService =
+                context.getServiceImpl(IRestApiService.class);
+        restService.addRestletRoutable(new DebugEventRoutable());
+        DebugEventAppender.setDebugEventServiceImpl(this);
+    }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/DebugEventAppender.java b/src/main/java/net/floodlightcontroller/debugevent/DebugEventAppender.java
new file mode 100644
index 0000000..3429675
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/DebugEventAppender.java
@@ -0,0 +1,96 @@
+package net.floodlightcontroller.debugevent;
+
+import net.floodlightcontroller.debugevent.IDebugEventService.EventColumn;
+import net.floodlightcontroller.debugevent.IDebugEventService.EventFieldType;
+import net.floodlightcontroller.debugevent.IDebugEventService.EventType;
+import net.floodlightcontroller.debugevent.IDebugEventService.MaxEventsRegistered;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.UnsynchronizedAppenderBase;
+
+public class DebugEventAppender<E> extends UnsynchronizedAppenderBase<E> {
+    static IDebugEventService debugEvent;
+    static IEventUpdater<WarnErrorEvent> evWarnError;
+    static Thread debugEventRegistryTask = new Thread() {
+        @Override
+        public void run() {
+            while(DebugEventAppender.debugEvent == null) {
+                try {
+                    Thread.sleep(100);
+                } catch (InterruptedException e) {
+                    return;
+                }
+            }
+            //safe to register debugEvent
+            registerDebugEventQueue();
+        }
+    };
+
+    @Override
+    public void start() {
+        DebugEventAppender.debugEventRegistryTask.start();
+        super.start();
+    }
+
+    public static void setDebugEventServiceImpl(IDebugEventService debugEvent) {
+        DebugEventAppender.debugEvent = debugEvent;
+        // It is now ok to register an event Q - but letting this thread go
+        // since it was called from a startUp() routine
+    }
+
+    /**
+     * The logging system calls append for every log message. This method filters
+     * out the WARN and ERROR message and adds to a debug event queue that can
+     * be accessed via cli or rest-api or gui.
+     */
+    @Override
+    protected void append(E eventObject) {
+        if (!isStarted()) {
+            return;
+        }
+        if (evWarnError != null) {
+            ILoggingEvent ev = ((ILoggingEvent) eventObject);
+            if (ev.getLevel().equals(Level.ERROR) || ev.getLevel().equals(Level.WARN)) {
+                evWarnError.updateEventWithFlush(
+                      new WarnErrorEvent(ev.getFormattedMessage(), ev.getLevel(),
+                                         ev.getThreadName(), ev.getLoggerName()));
+            }
+        }
+    }
+
+    private static void registerDebugEventQueue() {
+        try {
+            evWarnError = debugEvent.registerEvent("net.floodlightcontroller.core",
+                                     "warn-error-queue",
+                                     "all WARN and ERROR logs",
+                                     EventType.ALWAYS_LOG, WarnErrorEvent.class,
+                                     100);
+        } catch (MaxEventsRegistered e) {
+            e.printStackTrace();
+        }
+
+    }
+
+    public static class WarnErrorEvent {
+        @EventColumn(name = "message", description = EventFieldType.STRING)
+        String message;
+
+        @EventColumn(name = "level", description = EventFieldType.OBJECT)
+        Level level;
+
+        @EventColumn(name = "threadName", description = EventFieldType.STRING)
+        String threadName;
+
+        @EventColumn(name = "logger", description = EventFieldType.OBJECT)
+        String logger;
+
+        public WarnErrorEvent(String message, Level level, String threadName,
+                              String logger) {
+            this.message = message;
+            this.level = level;
+            this.threadName = threadName;
+            this.logger = logger;
+        }
+    }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/Event.java b/src/main/java/net/floodlightcontroller/debugevent/Event.java
new file mode 100644
index 0000000..b3e284e
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/Event.java
@@ -0,0 +1,233 @@
+package net.floodlightcontroller.debugevent;
+
+import java.lang.ref.SoftReference;
+import java.lang.reflect.Field;
+import java.text.SimpleDateFormat;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.floodlightcontroller.debugevent.IDebugEventService.EventColumn;
+import net.onrc.onos.core.packet.IPv4;
+import net.onrc.onos.core.util.SwitchPort;
+
+import org.openflow.protocol.OFFlowMod;
+import org.openflow.util.HexString;
+
+public class Event {
+    long timestamp;
+    long threadId;
+    String threadName;
+    Object eventData;
+    private Map<String, String> returnMap;
+
+    public Event(long timestamp, long threadId, String threadName, Object eventData) {
+        super();
+        this.timestamp = timestamp;
+        this.threadId = threadId;
+        this.threadName = threadName;
+        this.eventData = eventData;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public void setTimestamp(long timestamp) {
+        this.timestamp = timestamp;
+    }
+
+    public long getThreadId() {
+        return threadId;
+    }
+
+    public void setThreadId(long threadId) {
+        this.threadId = threadId;
+    }
+
+    public String getThreadName() {
+        return threadName;
+    }
+
+    public void setThreadName(String threadName) {
+        this.threadName = threadName;
+    }
+
+    public Object geteventData() {
+        return eventData;
+    }
+
+    public void seteventData(Object eventData) {
+        this.eventData = eventData;
+    }
+
+    /**
+     * If an old event (eg. popped from a circular buffer) is being re-used for
+     * storing a new event, it is very important to clear the cached formatted
+     * event, so that the formatting can be redone with the new event data.
+     * Otherwise it will appear as if the circular buffer is not getting updated
+     * at all as old (cached) formatted event is delivered to the user.
+     */
+    public void nullifyCachedFormattedEvent() {
+        this.returnMap = null;
+    }
+
+    @Override
+    public String toString() {
+        return "Event [timestamp=" + timestamp + ", threadId=" + threadId
+               + ", eventData=" + eventData.toString() + "]";
+    }
+
+    public Map<String, String> getFormattedEvent(Class<?> eventClass, String moduleEventName) {
+        if (eventClass == null || !eventClass.equals(eventData.getClass())) {
+            returnMap = new HashMap<String, String>();
+            returnMap.put("Error", "null event data or event-class does not match event-data");
+            return returnMap;
+        }
+        // return cached value if there is one
+        if (returnMap != null)
+            return returnMap;
+
+        returnMap = new HashMap<String, String>();
+        returnMap.put("Timestamp", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
+                                            .format(timestamp));
+        returnMap.put("Thread Id", String.valueOf(threadId));
+        returnMap.put("Thread Name", String.valueOf(threadName));
+        customFormat(eventClass, eventData, returnMap);
+        return returnMap;
+    }
+
+    private void customFormat(Class<?> clazz, Object eventData,
+                              Map<String, String> retMap) {
+        for (Field f : clazz.getDeclaredFields()) {
+            EventColumn ec = f.getAnnotation(EventColumn.class);
+            if (ec == null) continue;
+            f.setAccessible(true);
+            try {
+                Object obj =  f.get(eventData);
+
+                switch(ec.description()) {
+                    case DPID:
+                        retMap.put(ec.name(), HexString.toHexString((Long) obj));
+                        break;
+                    case MAC:
+                        retMap.put(ec.name(), HexString.toHexString((Long) obj, 6));
+                        break;
+                    case IPv4:
+                        retMap.put(ec.name(), net.onrc.onos.core.packet.IPv4.fromIPv4Address((Integer) obj));
+                        break;
+                    case FLOW_MOD_FLAGS:
+                        int flags = (Integer)obj;
+                        StringBuilder builder = new StringBuilder();
+                        if (flags == 0) {
+                            builder.append("None");
+                        }
+                        else {
+                            if ((flags & OFFlowMod.OFPFF_SEND_FLOW_REM) != 0) {
+                                builder.append("SEND_FLOW_REM ");
+                            }
+                            if ((flags & OFFlowMod.OFPFF_CHECK_OVERLAP) != 0) {
+                                builder.append("CHECK_OVERLAP ");
+                            }
+                            if ((flags & OFFlowMod.OFPFF_EMERG) != 0) {
+                                builder.append("EMERG ");
+                            }
+                        }
+                        retMap.put(ec.name(), builder.toString());
+                        break;
+                    case LIST_IPV4:
+                        @SuppressWarnings("unchecked")
+                        List<Integer> ipv4Addresses = (List<Integer>)obj;
+                        StringBuilder ipv4AddressesStr = new StringBuilder();
+                        if (ipv4Addresses.size() == 0) {
+                            ipv4AddressesStr.append("--");
+                        } else {
+                            for (Integer ipv4Addr : ipv4Addresses) {
+                                ipv4AddressesStr.append(IPv4.fromIPv4Address(ipv4Addr.intValue()));
+                                ipv4AddressesStr.append(" ");
+                            }
+                        }
+                        retMap.put(ec.name(), ipv4AddressesStr.toString());
+                        break;
+                    case LIST_ATTACHMENT_POINT:
+                        @SuppressWarnings("unchecked")
+                        List<SwitchPort> aps = (List<SwitchPort>)obj;
+                        StringBuilder apsStr = new StringBuilder();
+                        if (aps.size() == 0) {
+                            apsStr.append("--");
+                        } else {
+                            for (SwitchPort ap : aps) {
+                                apsStr.append(HexString.toHexString(ap.dpid().value()));
+                                apsStr.append("/");
+                                apsStr.append(ap.port().value());
+                                apsStr.append(" ");
+                            }
+                        }
+                        retMap.put(ec.name(), apsStr.toString());
+                        break;
+                    case LIST_OBJECT:
+                        @SuppressWarnings("unchecked")
+                        List<Object> obl = (List<Object>)obj;
+                        StringBuilder sbldr = new StringBuilder();
+                        if (obl.size() == 0) {
+                            sbldr.append("--");
+                        } else {
+                            for (Object o : obl) {
+                                sbldr.append(o.toString());
+                                sbldr.append(" ");
+                            }
+                        }
+                        retMap.put(ec.name(), sbldr.toString());
+                        break;
+                    case SREF_LIST_OBJECT:
+                        @SuppressWarnings("unchecked")
+                        SoftReference<List<Object>> srefListObj =
+                            (SoftReference<List<Object>>)obj;
+                        List<Object> ol = srefListObj.get();
+                        if (ol != null) {
+                            StringBuilder sb = new StringBuilder();
+                            if (ol.size() == 0) {
+                                sb.append("--");
+                            } else {
+                                for (Object o : ol) {
+                                    sb.append(o.toString());
+                                    sb.append(" ");
+                                }
+                            }
+                            retMap.put(ec.name(), sb.toString());
+                        } else {
+                            retMap.put(ec.name(), "-- reference not available --");
+                        }
+                        break;
+                    case SREF_OBJECT:
+                        @SuppressWarnings("unchecked")
+                        SoftReference<Object> srefObj = (SoftReference<Object>)obj;
+                        if (srefObj == null) {
+                            retMap.put(ec.name(), "--");
+                        } else {
+                            Object o = srefObj.get();
+                            if (o != null) {
+                                retMap.put(ec.name(), o.toString());
+                            } else {
+                                retMap.put(ec.name(),
+                                           "-- reference not available --");
+                            }
+                        }
+                        break;
+                    case STRING:
+                    case OBJECT:
+                    case PRIMITIVE:
+                    default:
+                        retMap.put(ec.name(), obj.toString());
+                }
+            } catch (ClassCastException e) {
+                retMap.put("Error", e.getMessage());
+            } catch (IllegalArgumentException e) {
+                retMap.put("Error", e.getMessage());
+            } catch (IllegalAccessException e) {
+                retMap.put("Error", e.getMessage());
+            }
+        }
+    }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/IDebugEventService.java b/src/main/java/net/floodlightcontroller/debugevent/IDebugEventService.java
new file mode 100644
index 0000000..d54e9f6
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/IDebugEventService.java
@@ -0,0 +1,189 @@
+package net.floodlightcontroller.debugevent;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.List;
+import java.util.Map;
+
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.debugevent.DebugEvent.EventInfo;
+
+public interface IDebugEventService extends IFloodlightService {
+
+    /**
+     * Different event types. Events that are meant to be logged on demand
+     * need to be separately enabled/disabled.
+     */
+    public enum EventType {
+        ALWAYS_LOG,
+        LOG_ON_DEMAND
+    }
+
+    /**
+     * Describes the type of field obtained from reflection
+     */
+    enum EventFieldType {
+        DPID, IPv4, MAC, STRING, OBJECT, PRIMITIVE, LIST_IPV4,
+        LIST_ATTACHMENT_POINT, LIST_OBJECT, SREF_LIST_OBJECT, SREF_OBJECT,
+        FLOW_MOD_FLAGS
+    }
+
+    /**
+     * EventColumn is the only annotation given to the fields of the event
+     * when updating an event.
+     */
+    @Target(ElementType.FIELD)
+    @Retention(RetentionPolicy.RUNTIME)
+    public @interface EventColumn {
+        String name() default "param";
+        EventFieldType description() default EventFieldType.PRIMITIVE;
+    }
+
+    /**
+     * Debug Event Qualifiers
+     */
+    public static final String EV_MDATA_WARN = "warn";
+    public static final String EV_MDATA_ERROR = "error";
+
+    /**
+     *  A limit on the maximum number of events that can be created
+     */
+    public static final int MAX_EVENTS = 2000;
+
+    /**
+     * Public class for information returned in response to rest API calls.
+     */
+    public class DebugEventInfo {
+        EventInfo eventInfo;
+        List<Map<String,String>> events;
+
+        public DebugEventInfo(EventInfo eventInfo,
+                              List<Map<String, String>> eventHistory) {
+            this.eventInfo = eventInfo;
+            this.events = eventHistory;
+        }
+
+        public EventInfo getEventInfo() {
+            return eventInfo;
+        }
+
+        public List<Map<String,String>> getEvents() {
+            return events;
+        }
+    }
+
+    /**
+    * exception thrown when MAX_EVENTS have been registered
+    */
+    public class MaxEventsRegistered extends Exception {
+        private static final long serialVersionUID = 2609587082227510262L;
+    }
+
+    /**
+     * Register an event for debugging.
+     *
+     * @param moduleName       module registering event eg. linkdiscovery, virtualrouting.
+     * @param eventName        name given to event.
+     * @param eventDescription A descriptive string describing the event.
+     * @param eventType        EventType for this event. On-demand events have to
+     *                         be explicitly enabled using other methods in this API
+     * @param eventClass       A user defined class that annotates the fields
+     *                         with @EventColumn. This class specifies the
+     *                         fields/columns for this event.
+     * @param bufferCapacity   Number of events to store for this event in a circular
+     *                         buffer. Older events will be discarded once the
+     *                         buffer is full.
+     * @param metaData         variable arguments that qualify an event
+     *                         eg. EV_MDATA_WARN, EV_MDATA_ERROR etc. See Debug Event Qualifiers
+     * @return                 IEventUpdater with update methods that can be used to
+     *                         update an event of the given eventClass
+     * @throws MaxEventsRegistered
+     */
+    public <T> IEventUpdater<T> registerEvent(String moduleName, String eventName,
+                                              String eventDescription,
+                                              EventType eventType,
+                                              Class<T> eventClass,
+                                              int bufferCapacity,
+                                              String... metaData)
+                                                      throws MaxEventsRegistered;
+
+    /**
+     * Update the global event stores with values from the thread local stores. This
+     * method is not typically intended for use by any module. It's typical usage is from
+     * floodlight core for events that happen in the packet processing pipeline.
+     * For other rare events, flushEvents should be called.
+     */
+    public void flushEvents();
+
+    /**
+     * Determine if eventName is a registered event for a given moduleName
+     */
+    public boolean containsModuleEventName(String moduleName, String eventName);
+
+    /**
+     * Determine if any events have been registered for module of name moduleName
+     */
+    public boolean containsModuleName(String moduleName);
+
+    /**
+     * Get event history for all events. This call can be expensive as it
+     * formats the event histories for all events.
+     *
+     * @return  a list of all event histories or an empty list if no events have
+     *          been registered
+     */
+    public List<DebugEventInfo> getAllEventHistory();
+
+    /**
+     * Get event history for all events registered for a given moduleName
+     *
+     * @return  a list of all event histories for all events registered for the
+     *          the module or an empty list if there are no events for this module
+     */
+    public List<DebugEventInfo> getModuleEventHistory(String moduleName);
+
+    /**
+     * Get event history for a single event
+     *
+     * @param  moduleName  registered module name
+     * @param  eventName   registered event name for moduleName
+     * @param  last        last X events
+     * @return DebugEventInfo for that event, or null if the moduleEventName
+     *         does not correspond to a registered event.
+     */
+    public DebugEventInfo getSingleEventHistory(String moduleName, String eventName, int last);
+
+    /**
+     * Wipe out all event history for all registered events
+     */
+    public void resetAllEvents();
+
+    /**
+     * Wipe out all event history for all events registered for a specific module
+     *
+     * @param moduleName  registered module name
+     */
+    public void resetAllModuleEvents(String moduleName);
+
+    /**
+     * Wipe out event history for a single event
+     * @param  moduleName  registered module name
+     * @param  eventName   registered event name for moduleName
+     */
+    public void resetSingleEvent(String moduleName, String eventName);
+
+    /**
+     * Retrieve a list of moduleNames registered for debug events or an empty
+     * list if no events have been registered in the system
+     */
+    public List<String> getModuleList();
+
+    /**
+     * Returns a list of all events registered for a specific moduleName
+     * or a empty list
+     */
+    public List<String> getModuleEventList(String moduleName);
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/IEventUpdater.java b/src/main/java/net/floodlightcontroller/debugevent/IEventUpdater.java
new file mode 100644
index 0000000..7aec38f
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/IEventUpdater.java
@@ -0,0 +1,30 @@
+package net.floodlightcontroller.debugevent;
+
+/**
+ * eventUPdater is used to log events for pre-registered events.
+ */
+public interface IEventUpdater<T> {
+
+    /**
+     * Logs the instance of the event thread-locally. Flushing to the global
+     * circular buffer for this event is delayed resulting in better performance.
+     * This method should typically be used by those events that happen in the
+     * packet processing pipeline
+     *
+     * @param event    an instance of the user-defined event of type T
+     */
+    public void updateEventNoFlush(T event);
+
+    /**
+     * Logs the instance of the event thread-locally and immediated flushes
+     * to the global circular buffer for this event.
+     * This method should typically be used by those events that happen
+     * outside the packet processing pipeline
+     *
+     * @param event    an instance of the user-defined event of type T
+     */
+    public void updateEventWithFlush(T event);
+
+
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/NullDebugEvent.java b/src/main/java/net/floodlightcontroller/debugevent/NullDebugEvent.java
new file mode 100644
index 0000000..7d479ea
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/NullDebugEvent.java
@@ -0,0 +1,135 @@
+package net.floodlightcontroller.debugevent;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.FloodlightModuleException;
+import net.floodlightcontroller.core.module.IFloodlightModule;
+import net.floodlightcontroller.core.module.IFloodlightService;
+
+public class NullDebugEvent implements IFloodlightModule, IDebugEventService {
+
+
+    @Override
+    public void flushEvents() {
+
+    }
+
+    @Override
+    public Collection<Class<? extends IFloodlightService>>
+            getModuleServices() {
+        Collection<Class<? extends IFloodlightService>> services =
+                new ArrayList<Class<? extends IFloodlightService>>(1);
+        services.add(IDebugEventService.class);
+        return services;
+    }
+
+    @Override
+    public Map<Class<? extends IFloodlightService>, IFloodlightService>
+            getServiceImpls() {
+        Map<Class<? extends IFloodlightService>,
+        IFloodlightService> m =
+            new HashMap<Class<? extends IFloodlightService>,
+                IFloodlightService>();
+        m.put(IDebugEventService.class, this);
+        return m;
+    }
+
+    @Override
+    public Collection<Class<? extends IFloodlightService>>
+            getModuleDependencies() {
+        return null;
+    }
+
+    @Override
+    public void init(FloodlightModuleContext context)
+            throws FloodlightModuleException {
+
+    }
+
+    @Override
+    public void startUp(FloodlightModuleContext context)
+            throws FloodlightModuleException {
+
+    }
+
+    @Override
+    public boolean containsModuleEventName(String moduleName, String eventName) {
+        return false;
+    }
+
+    @Override
+    public boolean containsModuleName(String moduleName) {
+        return false;
+    }
+
+    @Override
+    public List<DebugEventInfo> getAllEventHistory() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<DebugEventInfo> getModuleEventHistory(String param) {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public DebugEventInfo getSingleEventHistory(String moduleName, String eventName,
+                                                int last) {
+        return null;
+    }
+
+    @Override
+    public void resetAllEvents() {
+
+    }
+
+    @Override
+    public void resetAllModuleEvents(String moduleName) {
+
+    }
+
+    @Override
+    public void resetSingleEvent(String moduleName, String eventName) {
+
+    }
+
+    @Override
+    public <T> IEventUpdater<T>
+            registerEvent(String moduleName, String eventName,
+                          String eventDescription, EventType eventType,
+                          Class<T> eventClass, int bufferCapacity,
+                          String... metaData) throws MaxEventsRegistered {
+        return new NullEventImpl<T>();
+    }
+
+    public class NullEventImpl<T> implements IEventUpdater<T> {
+
+        @Override
+        public void updateEventNoFlush(Object event) {
+
+        }
+
+        @Override
+        public void updateEventWithFlush(Object event) {
+
+        }
+
+    }
+
+    @Override
+    public List<String> getModuleList() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<String> getModuleEventList(String moduleName) {
+        return Collections.emptyList();
+    }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventResource.java b/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventResource.java
new file mode 100644
index 0000000..a3f06ce
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventResource.java
@@ -0,0 +1,322 @@
+package net.floodlightcontroller.debugevent.web;
+
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.floodlightcontroller.debugevent.IDebugEventService.DebugEventInfo;
+import net.floodlightcontroller.debugevent.IDebugEventService.EventType;
+
+import org.restlet.resource.Get;
+import org.restlet.resource.Post;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Web interface for Debug Events
+ *
+ * @author Saurav
+ */
+public class DebugEventResource extends DebugEventResourceBase {
+    protected static Logger logger =
+            LoggerFactory.getLogger(DebugEventResource.class);
+
+    /**
+     * The output JSON model that contains the counter information
+     */
+    public static class DebugEventInfoOutput {
+        protected class DEInfo {
+            private final boolean enabled;
+            private final int bufferCapacity;
+            private final EventType eventType;
+            private final String eventDesc;
+            private final String eventName;
+            private final String moduleName;
+            private final String[] metaData;
+            private final List<Map<String,String>> eventHistory;
+
+            DEInfo(DebugEventInfo dei) {
+                this.moduleName = dei.getEventInfo().getModuleName();
+                this.eventName = dei.getEventInfo().getEventName();
+                this.eventDesc = dei.getEventInfo().getEventDesc();
+                this.metaData = dei.getEventInfo().getMetaData();
+                this.enabled = dei.getEventInfo().isEnabled();
+                this.eventType = dei.getEventInfo().getEtype();
+                this.bufferCapacity = dei.getEventInfo().getBufferCapacity();
+                this.eventHistory = dei.getEvents();
+            }
+            public boolean isEnabled() {
+                return enabled;
+            }
+            public int getBufferCapacity() {
+                return bufferCapacity;
+            }
+            public String getEventDesc() {
+                return eventDesc;
+            }
+            public String getEventName() {
+                return eventName;
+            }
+            public String getModuleName() {
+                return moduleName;
+            }
+            public String[] getMetaData() {
+                return metaData;
+            }
+            public EventType getEventType() {
+                return eventType;
+            }
+            public List<Map<String,String>> getEventHistory() {
+                return eventHistory;
+            }
+
+        }
+
+        public Map<String, DEInfo> eventMap = null;
+        public List<String> names = null;
+        public String error = null;
+
+        DebugEventInfoOutput(boolean getList) {
+            if (!getList) {
+                eventMap = new HashMap<String, DEInfo>();
+            }
+        }
+        public Map<String, DEInfo> getEventMap() {
+            return eventMap;
+        }
+        public List<String> getNames() {
+            return names;
+        }
+        public String getError() {
+            return error;
+        }
+
+    }
+
+    public enum Option {
+        ALL, ONE_MODULE, ONE_MODULE_EVENT, ERROR_BAD_MODULE_NAME, ERROR_BAD_PARAM,
+        ERROR_BAD_MODULE_EVENT_NAME
+    }
+
+    public static class DebugEventPost {
+        public Boolean reset;
+
+        public Boolean getReset() {
+            return reset;
+        }
+        public void setReset(Boolean reset) {
+            this.reset = reset;
+        }
+    }
+
+    public static class ResetOutput {
+        String error = null;
+
+        public String getError() {
+            return error;
+        }
+        public void setError(String error) {
+            this.error = error;
+        }
+    }
+
+    /**
+     * Reset events
+     *
+     * If using curl:
+     * curl -X POST -d {\"reset\":true} -H "Content-Type: application/json" URL
+     * where URL must be in one of the following forms for resetting registered events:
+     * "http://{controller-hostname}:8080/wm/debugevent/
+     * "http://{controller-hostname}:8080/wm/debugevent/{param1}
+     * "http://{controller-hostname}:8080/wm/debugevent/{param1}/{param2}
+     *
+     * Not giving {param1} will reset all events
+     * {param1} can be 'all' or the name of a module. The former case will reset
+     * all events, while the latter will reset all events for the moduleName (if
+     * param2 is null).{param2} must be an eventName for the given moduleName to
+     * reset a specific event.
+     */
+    @Post
+    public ResetOutput postHandler(DebugEventPost postData) {
+        ResetOutput output = new ResetOutput();
+        String param1 = (String)getRequestAttributes().get("param1");
+        String param2 = (String)getRequestAttributes().get("param2");
+
+        if (postData.getReset() != null && postData.getReset()) {
+            Option choice = Option.ERROR_BAD_PARAM;
+
+            if (param1 == null) {
+                param1 = "all";
+                choice = Option.ALL;
+            } else if (param1.equals("all")) {
+                choice = Option.ALL;
+            } else if (param2 == null) {
+                boolean isRegistered = debugEvent.containsModuleName(param1);
+                if (isRegistered) {
+                    choice = Option.ONE_MODULE;
+                } else {
+                    choice = Option.ERROR_BAD_MODULE_NAME;
+                }
+            } else {
+                // differentiate between disabled and non-existing events
+                boolean isRegistered = debugEvent.containsModuleEventName(param1, param2);
+                if (isRegistered) {
+                    choice = Option.ONE_MODULE_EVENT;
+                } else {
+                    choice = Option.ERROR_BAD_MODULE_EVENT_NAME;
+                }
+            }
+
+            switch (choice) {
+                case ALL:
+                    debugEvent.resetAllEvents();
+                    break;
+                case ONE_MODULE:
+                    debugEvent.resetAllModuleEvents(param1);
+                    break;
+                case ONE_MODULE_EVENT:
+                    debugEvent.resetSingleEvent(param1, param2);
+                    break;
+                case ERROR_BAD_MODULE_NAME:
+                    output.error = "Module name has no corresponding registered events";
+                    break;
+                case ERROR_BAD_MODULE_EVENT_NAME:
+                    output.error = "Event not registered";
+                    break;
+                case ERROR_BAD_PARAM:
+                    output.error = "Bad param";
+            }
+        }
+
+        return output;
+
+    }
+
+    /**
+     * Return the debug event data for the get rest-api call
+     *
+     * URL must be in one of the following forms for retrieving a list
+     * moduleNames    "http://{controller-hostname}:8080/wm/debugevent/
+     * counterNames   "http://{controller-hostname}:8080/wm/debugevent/{moduleName}
+     *
+     * URL must be in one of the following forms for retrieving event data:
+     * "http://{controller-hostname}:8080/wm/debugevent/{param1}
+     * "http://{controller-hostname}:8080/wm/debugevent/{param1}/{param2}
+     *
+     *  where {param1} must be one of (no quotes):
+     *       null                   if nothing is given then by default the list
+     *                              of all moduleNames is returned for which
+     *                              events have been registered
+     *       "all"                  can return value/info on all active events
+     *                              but is currently disallowed
+     *       "{moduleName}"         returns value/info on events for the specified module
+     *                              depending on the value of param2
+     *  and   {param2} must be one of (no quotes):
+     *       null                   returns all eventNames registered for the
+     *                              given moduleName (in param1)
+     *       "{eventName}"          returns value/info for specific event if it is active.
+     *
+     */
+    @Get("json")
+    public DebugEventInfoOutput handleEventInfoQuery() {
+        Option choice = Option.ERROR_BAD_PARAM;
+        DebugEventInfoOutput output;
+        String laststr = getQueryValue("last");
+        int last = Integer.MAX_VALUE;
+        try {
+            if (laststr != null)
+                last = Integer.valueOf(laststr);
+            if (last < 1) last = Integer.MAX_VALUE;
+        } catch (NumberFormatException e) {
+            output = new DebugEventInfoOutput(false);
+            output.error = "Expected an integer requesting last X events;" +
+                           " received " + laststr;
+            return output;
+        }
+        String param1 = (String)getRequestAttributes().get("param1");
+        String param2 = (String)getRequestAttributes().get("param2");
+
+        if (param1 == null) {
+            output = new DebugEventInfoOutput(true);
+            return listEvents(output);
+        } else if (param1.equals("all")) {
+            output = new DebugEventInfoOutput(false);
+            //populateEvents(debugEvent.getAllEventHistory(), output);
+            output.error = "Cannot retrieve all events - please select a specific event";
+            return output;
+        }
+
+        if (param2 == null) {
+            output = new DebugEventInfoOutput(true);
+            boolean isRegistered = debugEvent.containsModuleName(param1);
+            if (isRegistered) {
+                return listEvents(param1, output);
+            } else {
+                choice = Option.ERROR_BAD_MODULE_NAME;
+            }
+        } else if (param2.equals("all")) {
+            output = new DebugEventInfoOutput(false);
+            //choice = Option.ONE_MODULE;
+            output.error = "Cannot retrieve all events - please select a specific event";
+            return output;
+        } else {
+            // differentiate between disabled and non-existing events
+            boolean isRegistered = debugEvent.containsModuleEventName(param1, param2);
+            if (isRegistered) {
+                choice = Option.ONE_MODULE_EVENT;
+            } else {
+                choice = Option.ERROR_BAD_MODULE_EVENT_NAME;
+            }
+        }
+
+        output = new DebugEventInfoOutput(false);
+        switch (choice) {
+            case ONE_MODULE:
+                populateEvents(debugEvent.getModuleEventHistory(param1), output);
+                break;
+            case ONE_MODULE_EVENT:
+                populateSingleEvent(debugEvent.getSingleEventHistory(param1, param2, last),
+                                    output);
+                break;
+            case ERROR_BAD_MODULE_NAME:
+                output.error = "Module name has no corresponding registered events";
+                break;
+            case ERROR_BAD_MODULE_EVENT_NAME:
+                output.error = "Event not registered";
+                break;
+            case ERROR_BAD_PARAM:
+            default:
+                output.error = "Bad param";
+        }
+
+        return output;
+    }
+
+    private DebugEventInfoOutput listEvents(DebugEventInfoOutput output) {
+        output.names = debugEvent.getModuleList();
+        return output;
+    }
+
+    private DebugEventInfoOutput listEvents(String moduleName,
+                                            DebugEventInfoOutput output) {
+        output.names = debugEvent.getModuleEventList(moduleName);
+        return output;
+    }
+
+    private void populateSingleEvent(DebugEventInfo singleEventHistory,
+                                     DebugEventInfoOutput output) {
+        if (singleEventHistory != null) {
+            output.eventMap.put(singleEventHistory.getEventInfo().getModuleEventName(),
+                                output.new DEInfo(singleEventHistory));
+        }
+    }
+
+    private void populateEvents(List<DebugEventInfo> eventHistory,
+                                DebugEventInfoOutput output) {
+        if (eventHistory != null) {
+            for (DebugEventInfo de : eventHistory)
+                populateSingleEvent(de, output);
+        }
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventResourceBase.java b/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventResourceBase.java
new file mode 100644
index 0000000..964deeb
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventResourceBase.java
@@ -0,0 +1,18 @@
+package net.floodlightcontroller.debugevent.web;
+
+
+import net.floodlightcontroller.debugevent.IDebugEventService;
+
+import org.restlet.resource.ResourceException;
+import org.restlet.resource.ServerResource;
+
+public class DebugEventResourceBase extends ServerResource{
+    protected IDebugEventService debugEvent;
+
+    @Override
+    protected void doInit() throws ResourceException {
+        super.doInit();
+        debugEvent = (IDebugEventService)getContext().getAttributes().
+                get(IDebugEventService.class.getCanonicalName());
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventRoutable.java b/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventRoutable.java
new file mode 100644
index 0000000..d4ee7c6
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/debugevent/web/DebugEventRoutable.java
@@ -0,0 +1,27 @@
+package net.floodlightcontroller.debugevent.web;
+
+import org.restlet.Context;
+import org.restlet.Restlet;
+import org.restlet.routing.Router;
+
+import net.floodlightcontroller.restserver.RestletRoutable;
+
+public class DebugEventRoutable implements RestletRoutable {
+
+    @Override
+    public Restlet getRestlet(Context context) {
+        Router router = new Router(context);
+        router.attach("/{param1}/{param2}/", DebugEventResource.class);
+        router.attach("/{param1}/{param2}", DebugEventResource.class);
+        router.attach("/{param1}/", DebugEventResource.class);
+        router.attach("/{param1}", DebugEventResource.class);
+        router.attach("/", DebugEventResource.class);
+        return router;
+    }
+
+    @Override
+    public String basePath() {
+        return "/wm/debugevent";
+    }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/util/EnumBitmaps.java b/src/main/java/net/floodlightcontroller/util/EnumBitmaps.java
new file mode 100644
index 0000000..a7503eb
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/util/EnumBitmaps.java
@@ -0,0 +1,141 @@
+package net.floodlightcontroller.util;
+
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * A utility class to convert between integer based bitmaps for (OpenFlow)
+ * flags and Enum and EnumSet based representations.
+ *
+ * The enum used to represent individual flags needs to implement the
+ * BitmapableEnum interface.
+ *
+ * Example:
+ * {@code
+ *   int bitmap = 0x11; // OFPPC_PORT_DOWN | OFPPC_NO_STP
+ *   EnumSet<OFPortConfig> s = toEnumSet(OFPortConfig.class, bitmap);
+ *   // s will contain OFPPC_PORT_DOWN and OFPPC_NO_STP
+ * }
+ *
+ * {@code
+ *    EnumSet<OFPortConfig> s = EnumSet.of(OFPPC_NO_STP, OFPPC_PORT_DOWN);
+ *    int bitmap = toBitmap(s); // returns 0x11
+ * }
+ * @author gregor
+ *
+ */
+public class EnumBitmaps {
+    /**
+     * Enums used to represent individual flags needs to implement this
+     * interface
+     */
+    public interface BitmapableEnum {
+        /** Return the value in the bitmap that the enum constant represents.
+         * The returned value must have only a single bit set. E.g.,1<<3
+         */
+        int getValue();
+    }
+
+    /**
+     * Convert an integer bitmap to an EnumSet.
+     *
+     * See class description for example
+     * @param type The Enum class to use. Must implement BitmapableEnum
+     * @param bitmap The integer bitmap
+     * @return A newly allocated EnumSet representing the bits set in the
+     * bitmap
+     * @throws NullPointerException if type is null
+     * @throws IllegalArgumentException if any enum constant from type has
+     * more than one bit set.
+     * @throws IllegalArgumentException if the bitmap has any bits set not
+     * represented by an enum constant.
+     */
+    public static <E extends Enum<E> & BitmapableEnum>
+            EnumSet<E> toEnumSet(Class<E> type, int bitmap) {
+        if (type == null)
+            throw new NullPointerException("Given enum type must not be null");
+        EnumSet<E> s = EnumSet.noneOf(type);
+        // allSetBitmap will eventually have all valid bits for the given
+        // type set.
+        int allSetBitmap = 0;
+        for (E element: type.getEnumConstants()) {
+            if (Integer.bitCount(element.getValue()) != 1) {
+                String msg = String.format("The %s (%x) constant of the " +
+                        "enum %s is supposed to represent a bitmap entry but " +
+                        "has more than one bit set.",
+                        element.toString(), element.getValue(), type.getName());
+                throw new IllegalArgumentException(msg);
+            }
+            allSetBitmap |= element.getValue();
+            if ((bitmap & element.getValue()) != 0)
+                s.add(element);
+        }
+        if (((~allSetBitmap) & bitmap) != 0) {
+            // check if only valid flags are set in the given bitmap
+            String msg = String.format("The bitmap %x for enum %s has " +
+                    "bits set that are presented by any enum constant",
+                    bitmap, type.getName());
+            throw new IllegalArgumentException(msg);
+        }
+        return s;
+    }
+
+    /**
+     * Return the bitmap mask with all possible bits set. E.g., If a bitmap
+     * has the individual flags 0x1, 0x2, and 0x8 (note the missing 0x4) then
+     * the mask will be 0xb (1011 binary)
+     *
+     * @param type The Enum class to use. Must implement BitmapableEnum
+     * @throws NullPointerException if type is null
+     * @throws IllegalArgumentException if any enum constant from type has
+     * more than one bit set
+     * @return an integer with all possible bits for the given bitmap enum
+     * type set.
+     */
+    public static <E extends Enum<E> & BitmapableEnum>
+            int getMask(Class<E> type) {
+        if (type == null)
+            throw new NullPointerException("Given enum type must not be null");
+        // allSetBitmap will eventually have all valid bits for the given
+        // type set.
+        int allSetBitmap = 0;
+        for (E element: type.getEnumConstants()) {
+            if (Integer.bitCount(element.getValue()) != 1) {
+                String msg = String.format("The %s (%x) constant of the " +
+                        "enum %s is supposed to represent a bitmap entry but " +
+                        "has more than one bit set.",
+                        element.toString(), element.getValue(), type.getName());
+                throw new IllegalArgumentException(msg);
+            }
+            allSetBitmap |= element.getValue();
+        }
+        return allSetBitmap;
+    }
+
+    /**
+     * Convert the given EnumSet to the integer bitmap representation
+     * @param set The EnumSet to convert. The enum must implement
+     * BitmapableEnum
+     * @return the integer bitmap
+     * @throws IllegalArgumentException if an enum constant from the set (!) has
+     * more than one bit set
+     * @throws NullPointerException if the set is null
+     */
+    public static <E extends Enum<E> & BitmapableEnum>
+            int toBitmap(Set<E> set) {
+        if (set == null)
+            throw new NullPointerException("Given set must not be null");
+        int bitmap = 0;
+        for (E element: set) {
+            if (Integer.bitCount(element.getValue()) != 1) {
+                String msg = String.format("The %s (%x) constant in the set " +
+                        "is supposed to represent a bitmap entry but " +
+                        "has more than one bit set.",
+                        element.toString(), element.getValue());
+                throw new IllegalArgumentException(msg);
+            }
+            bitmap |= element.getValue();
+        }
+        return bitmap;
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/util/LinkedHashSetWrapper.java b/src/main/java/net/floodlightcontroller/util/LinkedHashSetWrapper.java
new file mode 100644
index 0000000..65865c0
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/util/LinkedHashSetWrapper.java
@@ -0,0 +1,33 @@
+package net.floodlightcontroller.util;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+
+import com.google.common.collect.ForwardingCollection;
+
+/**
+ * A simple wrapper / forwarder that forwards all calls to a LinkedHashSet.
+ * This wrappers sole reason for existence is to implement the
+ * OrderedCollection marker interface.
+ * @author gregor
+ *
+ */
+public class LinkedHashSetWrapper<E>
+        extends ForwardingCollection<E> implements OrderedCollection<E> {
+    private final Collection<E> delegate;
+
+    public LinkedHashSetWrapper() {
+        super();
+        this.delegate = new LinkedHashSet<E>();
+    }
+
+    public LinkedHashSetWrapper(Collection<? extends E> c) {
+        super();
+        this.delegate = new LinkedHashSet<E>(c);
+    }
+
+    @Override
+    protected Collection<E> delegate() {
+        return this.delegate;
+    }
+}
diff --git a/src/main/java/net/floodlightcontroller/util/LoadMonitor.java b/src/main/java/net/floodlightcontroller/util/LoadMonitor.java
new file mode 100644
index 0000000..5b234cd
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/util/LoadMonitor.java
@@ -0,0 +1,275 @@
+/**
+ *    Copyright 2013, Big Switch Networks, Inc.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *    not use this file except in compliance with the License. You may obtain
+ *    a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ **/
+
+package net.floodlightcontroller.util;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+
+import net.floodlightcontroller.core.annotations.LogMessageDocs;
+import net.floodlightcontroller.core.annotations.LogMessageDoc;
+
+public class LoadMonitor implements Runnable {
+
+    public enum LoadLevel {
+        OK,
+        HIGH,
+        VERYHIGH,
+    }
+
+    public LoadLevel getLoadLevel() {
+        return loadlevel ;
+    }
+
+    public double getLoad() {
+        return load ;
+    }
+
+    public static final int LOADMONITOR_SAMPLING_INTERVAL = 1000; // mili-sec
+    public static final double THRESHOLD_HIGH = 0.90;
+    public static final double THRESHOLD_VERYHIGH = 0.95;
+    public static final int MAX_LOADED_ITERATIONS = 5;
+    public static final int MAX_LOAD_HISTORY = 5;
+
+    protected volatile double load;
+    protected volatile LoadLevel loadlevel;
+    protected int itersLoaded;
+
+    protected boolean isLinux;
+    protected int numcores;
+    protected int jiffyNanos;
+    protected long[] lastNanos;
+    protected long[] lastIdle;
+    protected Logger log;
+
+    public LoadMonitor(Logger log_) {
+        log = log_;
+        loadlevel = LoadLevel.OK;
+        load = 0.0;
+        itersLoaded = 0;
+
+        lastNanos = new long[MAX_LOAD_HISTORY];
+        lastIdle = new long[MAX_LOAD_HISTORY];
+        for (int i=0 ; i<MAX_LOAD_HISTORY ; i++) {
+            lastNanos[i] = 0L;
+            lastIdle[i] = 0L;
+        }
+
+        isLinux = System.getProperty("os.name").equals("Linux");
+        numcores = 1;
+        jiffyNanos = 10 * 1000 * 1000;
+        if (isLinux) {
+            try {
+                numcores = Integer.parseInt(
+                        this.runcmd("/usr/bin/nproc"));
+                jiffyNanos = (1000 * 1000 * 1000) / Integer.parseInt(
+                        this.runcmd("/usr/bin/getconf CLK_TCK"));
+            }
+            catch (NumberFormatException ex) {
+                if (log != null) {
+                        // Log message documented on runcmd function
+                    log.error("Exception in inializing load monitor ", ex);
+                }
+                else {
+                    ex.printStackTrace();
+                }
+            }
+        }
+    }
+
+    @Override
+    @LogMessageDocs({
+        @LogMessageDoc(
+            message="System under very heavy load, dropping some packet-ins",
+            explanation="We detcted that the system was under very heavy" +
+                        "  load, dropping some packet-ins temporarily"),
+        @LogMessageDoc(
+            message="System under heavy load, dropping some new flows",
+            explanation="We detcted that the system was under heavy load," +
+                        " dropping some new flows temporarily")
+    })
+    public void run() {
+        if (!isLinux) return;
+
+        long currNanos = System.nanoTime();
+        long currIdle = this.readIdle();
+        for (int i=0 ; i < (MAX_LOAD_HISTORY - 1) ; i++) {
+            lastNanos[i] = lastNanos[i+1];
+            lastIdle[i] = lastIdle[i+1];
+        }
+        lastNanos[MAX_LOAD_HISTORY - 1] = currNanos;
+        lastIdle[MAX_LOAD_HISTORY - 1] = currIdle;
+
+        if (itersLoaded >= MAX_LOADED_ITERATIONS) {
+            loadlevel = LoadLevel.OK;
+            itersLoaded = 0;
+            return;
+        }
+
+        long nanos = lastNanos[MAX_LOAD_HISTORY - 1] - lastNanos[0];
+        long idle = lastIdle[MAX_LOAD_HISTORY - 1] - lastIdle[0];
+        load =
+            1.0 - ((double)(idle * jiffyNanos) / (double)(nanos * numcores));
+
+        if (load > THRESHOLD_VERYHIGH) {
+            loadlevel = LoadLevel.VERYHIGH;
+            itersLoaded += 1;
+            String msg = "System under very heavy load, dropping packet-ins.";
+
+            if (log != null) {
+                log.error(msg);
+            }
+            else {
+                System.out.println(msg);
+            }
+            return;
+        }
+
+        if (load > THRESHOLD_HIGH) {
+            loadlevel = LoadLevel.HIGH;
+            itersLoaded += 1;
+            String msg = "System under heavy load, dropping new flows.";
+
+            if (log != null) {
+                log.error(msg);
+            }
+            else {
+                System.out.println(msg);
+            }
+            return;
+        }
+
+        loadlevel = LoadLevel.OK;
+        itersLoaded = 0;
+        return;
+    }
+
+    @LogMessageDoc(
+        message="Exception in reading load monitor params, using defaults",
+        explanation="There was an error in inializing load monitor's props," +
+                    " using default parameters")
+    protected String runcmd(String cmd) {
+        String line;
+        StringBuilder ret = new StringBuilder();
+        try {
+            Process p = Runtime.getRuntime().exec(cmd);
+            BufferedReader input =
+                new BufferedReader(
+                new InputStreamReader(p.getInputStream()));
+            while ((line = input.readLine()) != null) {
+                ret.append(line);
+            }
+            input.close();
+            p.waitFor();
+        }
+        catch (InterruptedException ex) {
+            if (log != null) {
+                log.error("Exception in inializing load monitor ", ex);
+            }
+            else {
+                ex.printStackTrace();
+            }
+        }
+        catch (IOException ex) {
+            if (log != null) {
+                log.error("Exception in inializing load monitor ", ex);
+            }
+            else {
+                ex.printStackTrace();
+            }
+        }
+        return ret.toString();
+
+    }
+
+    protected long readIdle() {
+        long idle = 0;
+        FileInputStream fs = null;
+        BufferedReader reader = null;
+        try {
+            try {
+                fs = new FileInputStream("/proc/stat");
+                reader = new BufferedReader(new InputStreamReader(fs));
+                String line = reader.readLine();
+                if (line == null) throw new IOException("Empty file");
+                idle = Long.parseLong(line.split("\\s+")[4]);
+            } finally {
+                if (reader != null)
+                    reader.close();
+                if (fs != null)
+                    fs.close();
+            }
+        } catch (IOException ex) {
+            log.error("Error reading idle time from /proc/stat", ex);
+        }
+        return idle;
+
+    }
+
+    public ScheduledFuture<?> startMonitoring(ScheduledExecutorService ses)
+    {
+        ScheduledFuture<?> monitorTask =
+            ses.scheduleAtFixedRate(
+                this, 0,
+                LOADMONITOR_SAMPLING_INTERVAL, TimeUnit.MILLISECONDS);
+        return monitorTask;
+    }
+
+    /*
+     * For testing
+     */
+    public ScheduledFuture<?> printMonitoring(ScheduledExecutorService ses)
+    {
+        final LoadMonitor mon = this;
+        ScheduledFuture<?> monitorTask =
+            ses.scheduleAtFixedRate(
+                new Runnable() {
+                    public void run() {
+                        System.out.println(mon.getLoad());
+                    }
+                }, LOADMONITOR_SAMPLING_INTERVAL/2,
+                LOADMONITOR_SAMPLING_INTERVAL, TimeUnit.MILLISECONDS);
+        return monitorTask;
+    }
+
+    public static void main(String[] args) {
+        final LoadMonitor monitor = new LoadMonitor(null);
+        final ScheduledExecutorService scheduler =
+            Executors.newScheduledThreadPool(1);
+        final ScheduledFuture<?> monitorTask =
+            monitor.startMonitoring(scheduler);
+        final ScheduledFuture<?> printTask =
+            monitor.printMonitoring(scheduler);
+
+        // Run the tasks for 2 minutes
+        scheduler.schedule(
+            new Runnable() {
+                public void run() {
+                    monitorTask.cancel(true);
+                    printTask.cancel(true);
+                }
+            }, 5*60, TimeUnit.SECONDS);
+    }
+
+}
diff --git a/src/main/java/net/floodlightcontroller/util/OFMessageDamper.java b/src/main/java/net/floodlightcontroller/util/OFMessageDamper.java
index 5faf38e..90bee76 100644
--- a/src/main/java/net/floodlightcontroller/util/OFMessageDamper.java
+++ b/src/main/java/net/floodlightcontroller/util/OFMessageDamper.java
@@ -133,7 +133,8 @@
                          FloodlightContext cntx, boolean flush)
             throws IOException {
         if (!msgTypesToCache.contains(msg.getType())) {
-            sw.write(msg, cntx);
+            // XXX S commenting out old message writes
+		//sw.write(msg, cntx);
             if (flush) {
                 sw.flush();
             }
@@ -145,7 +146,8 @@
             // entry exists in cache. Dampening.
             return false;
         } else {
-            sw.write(msg, cntx);
+            // XXX S commenting out old message writes
+		// sw.write(msg, cntx);
             if (flush) {
                 sw.flush();
             }
diff --git a/src/main/java/net/floodlightcontroller/util/OrderedCollection.java b/src/main/java/net/floodlightcontroller/util/OrderedCollection.java
new file mode 100644
index 0000000..e302a72
--- /dev/null
+++ b/src/main/java/net/floodlightcontroller/util/OrderedCollection.java
@@ -0,0 +1,15 @@
+package net.floodlightcontroller.util;
+
+import java.util.Collection;
+
+/**
+ * A marker interface indicating that this Collection defines a particular
+ * iteration order. The details about the iteration order are specified by
+ * the concrete implementation.
+ * @author gregor
+ *
+ * @param <E>
+ */
+public interface OrderedCollection<E> extends Collection<E> {
+
+}
diff --git a/src/main/java/net/onrc/onos/api/registry/ILocalSwitchMastershipListener.java b/src/main/java/net/onrc/onos/api/registry/ILocalSwitchMastershipListener.java
deleted file mode 100644
index 2445b76..0000000
--- a/src/main/java/net/onrc/onos/api/registry/ILocalSwitchMastershipListener.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package net.onrc.onos.api.registry;
-
-// TODO: The "Role" enums should be moved to this file
-import net.floodlightcontroller.core.IFloodlightProviderService.Role;
-import net.onrc.onos.core.util.Dpid;
-
-/**
- * Switch mastership listener interface for controller role changes for
- * local switches.
- * <p/>
- * The interface can be used to track only switches that are connected
- * to this ONOS instance.
- */
-public interface ILocalSwitchMastershipListener {
-    /**
-     * The role of this controller has changed for a switch.
-     * <p/>
-     * This is the method that is called when the switch connects to the
-     * controller, and when the role of the controller has changed.
-     *
-     * @param dpid the DPID of the switch.
-     * @param role the new role of this controller for the switch.
-     */
-    void controllerRoleChanged(Dpid dpid, Role role);
-
-    /**
-     * The switch has disconnected, and it is not tracked anymore.
-     *
-     * @param dpid the DPID of the switch.
-     */
-    void switchDisconnected(Dpid dpid);
-}
diff --git a/src/main/java/net/onrc/onos/apps/sdnip/FlowCache.java b/src/main/java/net/onrc/onos/apps/sdnip/FlowCache.java
index 8c518d7..dc9182c 100644
--- a/src/main/java/net/onrc/onos/apps/sdnip/FlowCache.java
+++ b/src/main/java/net/onrc/onos/apps/sdnip/FlowCache.java
@@ -1,6 +1,5 @@
 package net.onrc.onos.apps.sdnip;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -14,7 +13,6 @@
 import org.openflow.protocol.OFFlowMod;
 import org.openflow.protocol.OFMessage;
 import org.openflow.protocol.OFPort;
-import org.openflow.util.HexString;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -27,19 +25,19 @@
 
     private final Comparator<OFFlowMod> cookieComparator =
             new Comparator<OFFlowMod>() {
-        @Override
-        public int compare(OFFlowMod fm1, OFFlowMod fm2) {
-            long difference = fm2.getCookie() - fm1.getCookie();
+                @Override
+                public int compare(OFFlowMod fm1, OFFlowMod fm2) {
+                    long difference = fm2.getCookie() - fm1.getCookie();
 
-            if (difference > 0) {
-                return 1;
-            } else if (difference < 0) {
-                return -1;
-            } else {
-                return 0;
-            }
-        }
-    };
+                    if (difference > 0) {
+                        return 1;
+                    } else if (difference < 0) {
+                        return -1;
+                    } else {
+                        return 0;
+                    }
+                }
+            };
 
     public FlowCache(IFloodlightProviderService floodlightProvider) {
         this.floodlightProvider = floodlightProvider;
@@ -83,11 +81,12 @@
             List<OFMessage> msgList = new ArrayList<OFMessage>(clones.size());
             msgList.addAll(clones);
 
-            try {
+            // XXX S commenting out the old message writes
+            /*try {
                 sw.write(msgList, null);
             } catch (IOException e) {
                 log.error("Error writing to switch", e);
-            }
+            }*/
         }
     }
 
@@ -124,11 +123,12 @@
             List<OFMessage> msgList = new ArrayList<OFMessage>(flowMods.size());
             msgList.addAll(flowMods);
 
-            try {
-                sw.write(msgList, null);
+            // XXX S commenting out old message writes
+            /*try {
+            sw.write(msgList, null);
             } catch (IOException e) {
                 log.error("Error writing to switch", e);
-            }
+            }*/
         }
     }
 
@@ -147,12 +147,14 @@
             List<OFMessage> messages = new ArrayList<OFMessage>(flowMods.size());
             messages.addAll(flowMods);
 
-            try {
-                sw.write(messages, null);
-            } catch (IOException e) {
-                log.error("Failure writing flow mods to switch {}",
-                        HexString.toHexString(sw.getId()));
-            }
+            // XXX S commenting out old message writes
+            /*            try {
+                        sw.write(messages, null);
+                        } catch (IOException e) {
+                            log.error("Failure writing flow mods to switch {}",
+                                    HexString.toHexString(sw.getId()));
+                        }
+            */
         }
     }
 
diff --git a/src/main/java/net/onrc/onos/apps/sdnip/SdnIp.java b/src/main/java/net/onrc/onos/apps/sdnip/SdnIp.java
index f4bf28d..135d0a7 100644
--- a/src/main/java/net/onrc/onos/apps/sdnip/SdnIp.java
+++ b/src/main/java/net/onrc/onos/apps/sdnip/SdnIp.java
@@ -17,6 +17,7 @@
 
 import net.floodlightcontroller.core.IFloodlightProviderService;
 import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.IOFSwitch.PortChangeType;
 import net.floodlightcontroller.core.IOFSwitchListener;
 import net.floodlightcontroller.core.module.FloodlightModuleContext;
 import net.floodlightcontroller.core.module.FloodlightModuleException;
@@ -68,6 +69,7 @@
 import org.openflow.protocol.action.OFAction;
 import org.openflow.protocol.action.OFActionOutput;
 import org.openflow.util.HexString;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -83,8 +85,7 @@
 import com.googlecode.concurrenttrees.radixinverted.InvertedRadixTree;
 
 public class SdnIp implements IFloodlightModule, ISdnIpService,
-        IArpRequester,
-        IOFSwitchListener, IConfigInfoService {
+        IArpRequester, IOFSwitchListener, IConfigInfoService {
 
     private static final Logger log = LoggerFactory.getLogger(SdnIp.class);
     private final CallerId callerId = new CallerId("SDNIP");
@@ -146,7 +147,7 @@
     // True when we have a full mesh of shortest paths between gateways
     private volatile boolean topologyReady = false;
 
-    //private SingletonTask topologyChangeDetectorTask;
+    // private SingletonTask topologyChangeDetectorTask;
 
     private SetMultimap<InetAddress, RibUpdate> prefixesWaitingOnArp;
 
@@ -156,7 +157,7 @@
 
     private Map<InetAddress, Path> pushedPaths;
     private Map<Prefix, Path> prefixToPath;
-    //  private Multimap<Prefix, PushedFlowMod> pushedFlows;
+    // private Multimap<Prefix, PushedFlowMod> pushedFlows;
     private Multimap<Prefix, FlowId> pushedFlowIds;
 
     private FlowCache flowCache;
@@ -245,8 +246,7 @@
 
     @Override
     public Collection<Class<? extends IFloodlightService>> getModuleServices() {
-        Collection<Class<? extends IFloodlightService>> l
-                = new ArrayList<>();
+        Collection<Class<? extends IFloodlightService>> l = new ArrayList<>();
         l.add(ISdnIpService.class);
         l.add(IConfigInfoService.class);
         return l;
@@ -254,8 +254,7 @@
 
     @Override
     public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() {
-        Map<Class<? extends IFloodlightService>, IFloodlightService> m
-                = new HashMap<>();
+        Map<Class<? extends IFloodlightService>, IFloodlightService> m = new HashMap<>();
         m.put(ISdnIpService.class, this);
         m.put(IConfigInfoService.class, this);
         return m;
@@ -263,8 +262,7 @@
 
     @Override
     public Collection<Class<? extends IFloodlightService>> getModuleDependencies() {
-        Collection<Class<? extends IFloodlightService>> l
-                = new ArrayList<>();
+        Collection<Class<? extends IFloodlightService>> l = new ArrayList<>();
         l.add(IFloodlightProviderService.class);
         l.add(IRestApiService.class);
         l.add(IControllerRegistryService.class);
@@ -289,11 +287,14 @@
         restApi = context.getServiceImpl(IRestApiService.class);
         proxyArp = context.getServiceImpl(IProxyArpService.class);
 
-        controllerRegistryService = context.getServiceImpl(IControllerRegistryService.class);
+        controllerRegistryService = context
+                .getServiceImpl(IControllerRegistryService.class);
         pathRuntime = context.getServiceImpl(IPathCalcRuntimeService.class);
 
-        //ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
-        //topologyChangeDetectorTask = new SingletonTask(executor, new TopologyChangeDetector());
+        // ScheduledExecutorService executor =
+        // Executors.newScheduledThreadPool(1);
+        // topologyChangeDetectorTask = new SingletonTask(executor, new
+        // TopologyChangeDetector());
 
         pathsWaitingOnArp = new HashMap<>();
         prefixesWaitingOnArp = Multimaps.synchronizedSetMultimap(
@@ -301,7 +302,7 @@
 
         pushedPaths = new HashMap<>();
         prefixToPath = new HashMap<>();
-//              pushedFlows = HashMultimap.<Prefix, PushedFlowMod>create();
+        // pushedFlows = HashMultimap.<Prefix, PushedFlowMod>create();
         pushedFlowIds = HashMultimap.create();
 
         flowCache = new FlowCache(floodlightProvider);
@@ -439,8 +440,10 @@
             RibEntry rib = bgpRoutes.put(prefix.toBinaryString(), update.getRibEntry());
 
             if (rib != null && !rib.equals(update.getRibEntry())) {
-                // There was an existing nexthop for this prefix. This update supersedes that,
-                // so we need to remove the old flows for this prefix from the switches
+                // There was an existing nexthop for this prefix. This update
+                // supersedes that,
+                // so we need to remove the old flows for this prefix from the
+                // switches
                 executeDeletePrefix(prefix, rib);
             }
 
@@ -527,11 +530,11 @@
     }
 
     /**
-     * Add a flow to match dst-IP prefix and rewrite MAC for one IP prefix
-     * to all other border switches.
+     * Add a flow to match dst-IP prefix and rewrite MAC for one IP prefix to
+     * all other border switches.
      */
     private void addPrefixFlows(Prefix prefix, Interface egressInterface,
-                                MACAddress nextHopMacAddress) {
+            MACAddress nextHopMacAddress) {
         log.debug("Adding flows for prefix {}, next hop mac {}",
                 prefix, nextHopMacAddress);
 
@@ -550,10 +553,11 @@
 
         // Create the DataPath object: dstSwitchPort
         SwitchPort dstPort =
-            new SwitchPort(new Dpid(egressInterface.getDpid()),
-                           new PortNumber(egressInterface.getPort()));
+                new SwitchPort(new Dpid(egressInterface.getDpid()),
+                        new PortNumber(egressInterface.getPort()));
 
-        // We only need one flow mod per switch, so pick one interface on each switch
+        // We only need one flow mod per switch, so pick one interface on each
+        // switch
         Map<Long, Interface> srcInterfaces = new HashMap<>();
         for (Interface intf : interfaces.values()) {
             if (!srcInterfaces.containsKey(intf.getDpid())
@@ -572,8 +576,8 @@
 
             // Create DataPath object: srcSwitchPort
             SwitchPort srcPort =
-                new SwitchPort(new Dpid(srcInterface.getDpid()),
-                               new PortNumber(srcInterface.getPort()));
+                    new SwitchPort(new Dpid(srcInterface.getDpid()),
+                            new PortNumber(srcInterface.getPort()));
 
             DataPath dataPath = new DataPath();
             dataPath.setSrcPort(srcPort);
@@ -620,9 +624,11 @@
         synchronized (this) {
             Prefix prefix = update.getPrefix();
 
-            //if (ptree.remove(prefix, update.getRibEntry())) {
-            // TODO check the change of logic here - remove doesn't check that the
-            // rib entry was what we expected (and we can't do this concurrently)
+            // if (ptree.remove(prefix, update.getRibEntry())) {
+            // TODO check the change of logic here - remove doesn't check that
+            // the
+            // rib entry was what we expected (and we can't do this
+            // concurrently)
             if (bgpRoutes.remove(prefix.toBinaryString())) {
                 /*
                  * Only delete flows if an entry was actually removed from the tree.
@@ -644,8 +650,8 @@
             Path path = prefixToPath.remove(prefix);
 
             if (path != null) {
-                //path could be null if we added to the Ptree but didn't push
-                //flows yet because we were waiting to resolve ARP
+                // path could be null if we added to the Ptree but didn't push
+                // flows yet because we were waiting to resolve ARP
 
                 path.decrementUsers();
                 if (path.getUsers() <= 0 && !path.isPermanent()) {
@@ -707,9 +713,8 @@
         }*/
     }
 
-
-    //TODO test next-hop changes
-    //TODO check delete/add synchronization
+    // TODO test next-hop changes
+    // TODO check delete/add synchronization
 
     /**
      * On startup, we need to calculate a full mesh of paths between all gateway
@@ -778,15 +783,16 @@
         flowPath.setFlowPathType(FlowPathType.FP_TYPE_SHORTEST_PATH);
         flowPath.setFlowPathUserState(FlowPathUserState.FP_USER_ADD);
 
-        // Insert the dest-mac based forwarding flow entry to the non-first-hop switches
+        // Insert the dest-mac based forwarding flow entry to the non-first-hop
+        // switches
         FlowPathFlags flowPathFlags = new FlowPathFlags();
         flowPathFlags.setFlags(FlowPathFlags.DISCARD_FIRST_HOP_ENTRY);
         flowPath.setFlowPathFlags(flowPathFlags);
 
         // Create the DataPath object: dstSwitchPort
         SwitchPort dstPort =
-            new SwitchPort(new Dpid(dstInterface.getDpid()),
-                           new PortNumber(dstInterface.getPort()));
+                new SwitchPort(new Dpid(dstInterface.getDpid()),
+                        new PortNumber(dstInterface.getPort()));
 
         for (Interface srcInterface : interfaces.values()) {
 
@@ -799,8 +805,8 @@
 
             // Create the DataPath object: srcSwitchPort
             SwitchPort srcPort =
-                new SwitchPort(new Dpid(srcInterface.getDpid()),
-                               new PortNumber(srcInterface.getPort()));
+                    new SwitchPort(new Dpid(srcInterface.getDpid()),
+                            new PortNumber(srcInterface.getPort()));
 
             DataPath dataPath = new DataPath();
             dataPath.setSrcPort(srcPort);
@@ -814,7 +820,8 @@
             flowPath.setFlowEntryMatch(flowEntryMatch);
 
             // NOTE: No need to add ACTION_OUTPUT. It is implied when creating
-            // Shortest Path Flow, and is always the last action for the Flow Entries
+            // Shortest Path Flow, and is always the last action for the Flow
+            // Entries
             log.debug("FlowPath of MAC based forwarding: {}", flowPath.toString());
             // TODO: Add the flow by using the new Path Intent framework
             /*
@@ -832,11 +839,11 @@
 
     @Override
     public void beginRoutingNew() {
-            setupBgpPathsNew();
+        setupBgpPathsNew();
 
-        //setupFullMesh();
+        // setupFullMesh();
 
-        //Suppress link discovery on external-facing router ports
+        // Suppress link discovery on external-facing router ports
 
         for (Interface intf : interfaces.values()) {
             linkDiscoveryService.disableDiscoveryOnPort(intf.getDpid(), intf.getPort());
@@ -851,37 +858,39 @@
     }
 
     /**
-     * Setup the Paths to the BGP Daemon.
-     *
-     * Run a loop for all of the bgpPeers
-     * Push flow from BGPd to the peer
-     * Push flow from peer to BGPd
-     * Parameters to pass to the intent are as follows:
-     *     String id,
-     *     long srcSwitch, long srcPort, long srcMac, int srcIP,
-     *     long dstSwitch, long dstPort, long dstMac, int dstIP
+     * Setup the Paths to the BGP Daemon. Run a loop for all of the bgpPeers
+     * Push flow from BGPd to the peer Push flow from peer to BGPd Parameters to
+     * pass to the intent are as follows: String id, long srcSwitch, long
+     * srcPort, long srcMac, int srcIP, long dstSwitch, long dstPort, long
+     * dstMac, int dstIP
      */
     private void setupBgpPathsNew() {
         IntentOperationList operations = new IntentOperationList();
         for (BgpPeer bgpPeer : bgpPeers.values()) {
             Interface peerInterface = interfaces.get(bgpPeer.getInterfaceName());
-            //Inet4Address.
+            // Inet4Address.
             int srcIP = InetAddresses.coerceToInteger(peerInterface.getIpAddress());
             int dstIP = InetAddresses.coerceToInteger(bgpPeer.getIpAddress());
-            String fwdIntentId = caller + ":" + controllerRegistryService.getNextUniqueId();
-            String bwdIntentId = caller + ":" + controllerRegistryService.getNextUniqueId();
+            String fwdIntentId = caller + ":"
+                    + controllerRegistryService.getNextUniqueId();
+            String bwdIntentId = caller + ":"
+                    + controllerRegistryService.getNextUniqueId();
             SwitchPort srcPort =
-                new SwitchPort(bgpdAttachmentPoint.dpid(),
-                               bgpdAttachmentPoint.port());
+                    new SwitchPort(bgpdAttachmentPoint.dpid(),
+                            bgpdAttachmentPoint.port());
             SwitchPort dstPort =
-                new SwitchPort(new Dpid(peerInterface.getDpid()),
-                               new PortNumber(peerInterface.getSwitchPort().port()));
+                    new SwitchPort(new Dpid(peerInterface.getDpid()),
+                            new PortNumber(peerInterface.getSwitchPort().port()));
             ShortestPathIntent fwdIntent = new ShortestPathIntent(fwdIntentId,
-                    srcPort.dpid().value(), srcPort.port().value(), ShortestPathIntent.EMPTYMACADDRESS, srcIP,
-                    dstPort.dpid().value(), dstPort.port().value(), ShortestPathIntent.EMPTYMACADDRESS, dstIP);
+                    srcPort.dpid().value(), srcPort.port().value(),
+                    ShortestPathIntent.EMPTYMACADDRESS, srcIP,
+                    dstPort.dpid().value(), dstPort.port().value(),
+                    ShortestPathIntent.EMPTYMACADDRESS, dstIP);
             ShortestPathIntent bwdIntent = new ShortestPathIntent(bwdIntentId,
-                    dstPort.dpid().value(), dstPort.port().value(), ShortestPathIntent.EMPTYMACADDRESS, dstIP,
-                    srcPort.dpid().value(), srcPort.port().value(), ShortestPathIntent.EMPTYMACADDRESS, srcIP);
+                    dstPort.dpid().value(), dstPort.port().value(),
+                    ShortestPathIntent.EMPTYMACADDRESS, dstIP,
+                    srcPort.dpid().value(), srcPort.port().value(),
+                    ShortestPathIntent.EMPTYMACADDRESS, srcIP);
             IntentOperation.Operator operator = IntentOperation.Operator.ADD;
             operations.add(operator, fwdIntent);
             operations.add(operator, bwdIntent);
@@ -914,11 +923,13 @@
             flowEntryMatch.enableEthernetFrameType(Ethernet.TYPE_IPV4);
 
             // Match both source address and dest address
-            IPv4Net dstIPv4Net = new IPv4Net(bgpPeer.getIpAddress().getHostAddress() + "/32");
+            IPv4Net dstIPv4Net = new IPv4Net(bgpPeer.getIpAddress().getHostAddress()
+                    + "/32");
 
             flowEntryMatch.enableDstIPv4Net(dstIPv4Net);
 
-            IPv4Net srcIPv4Net = new IPv4Net(peerInterface.getIpAddress().getHostAddress() + "/32");
+            IPv4Net srcIPv4Net = new IPv4Net(peerInterface.getIpAddress()
+                    .getHostAddress() + "/32");
             flowEntryMatch.enableSrcIPv4Net(srcIPv4Net);
 
             // Match TCP protocol
@@ -935,13 +946,13 @@
             DataPath dataPath = new DataPath();
 
             SwitchPort srcPort =
-                new SwitchPort(bgpdAttachmentPoint.dpid(),
-                               bgpdAttachmentPoint.port());
+                    new SwitchPort(bgpdAttachmentPoint.dpid(),
+                            bgpdAttachmentPoint.port());
             dataPath.setSrcPort(srcPort);
 
             SwitchPort dstPort =
-                new SwitchPort(new Dpid(peerInterface.getDpid()),
-                               new PortNumber(peerInterface.getSwitchPort().port()));
+                    new SwitchPort(new Dpid(peerInterface.getDpid()),
+                            new PortNumber(peerInterface.getSwitchPort().port()));
             dataPath.setDstPort(dstPort);
 
             flowPath.setDataPath(dataPath);
@@ -1070,13 +1081,10 @@
 
             // match ICMP protocol BGP -> Peer
             flowPath.setFlowId(new FlowId());
-
             flowEntryMatch.enableDstIPv4Net(dstIPv4Net);
             flowEntryMatch.enableSrcIPv4Net(srcIPv4Net);
             flowPath.setFlowEntryMatch(flowEntryMatch);
-
             flowPath.setDataPath(dataPath);
-
             log.debug("ICMP flowPath: {}", flowPath.toString());
 
             // TODO: Add the flow by using the new Path Intent framework
@@ -1111,11 +1119,14 @@
                 log.debug("Pushing path to {} at {} on {}",
                         path.getDstIpAddress().getHostAddress(), macAddress,
                         path.getDstInterface().getSwitchPort());
-                // These paths should always be to BGP peers. Paths to non-peers are
+                // These paths should always be to BGP peers. Paths to non-peers
+                // are
                 // handled once the first prefix is ready to push
                 if (pushedPaths.containsKey(path.getDstIpAddress())) {
-                    // A path already got pushed to this endpoint while we were waiting
-                    // for ARP. We'll copy over the permanent attribute if it is set on this path.
+                    // A path already got pushed to this endpoint while we were
+                    // waiting
+                    // for ARP. We'll copy over the permanent attribute if it is
+                    // set on this path.
                     if (path.isPermanent()) {
                         pushedPaths.get(path.getDstIpAddress()).setPermanent();
                     }
@@ -1135,9 +1146,12 @@
                 if (rib != null && rib.equals(update.getRibEntry())) {
                     log.debug("Pushing prefix {} next hop {}", update.getPrefix(),
                             rib.getNextHop().getHostAddress());
-                    // We only push prefix flows if the prefix is still in the ptree
-                    // and the next hop is the same as our update. The prefix could
-                    // have been removed while we were waiting for the ARP, or the
+                    // We only push prefix flows if the prefix is still in the
+                    // ptree
+                    // and the next hop is the same as our update. The prefix
+                    // could
+                    // have been removed while we were waiting for the ARP, or
+                    // the
                     // next hop could have changed.
                     executeRibAdd(update);
                 } else {
@@ -1247,7 +1261,7 @@
         setupBgpPaths();
         setupFullMesh();
 
-        //Suppress link discovery on external-facing router ports
+        // Suppress link discovery on external-facing router ports
         for (Interface intf : interfaces.values()) {
             linkDiscoveryService.disableDiscoveryOnPort(intf.getDpid(), intf.getPort());
         }
@@ -1285,7 +1299,8 @@
         */
     }
 
-    // Actually we only need to go half way round to verify full mesh connectivity
+    // Actually we only need to go half way round to verify full mesh
+    // connectivity
     private void checkTopologyReady() {
         for (Interface dstInterface : interfaces.values()) {
             for (Interface srcInterface : interfaces.values()) {
@@ -1329,25 +1344,25 @@
                 try {
                     RibUpdate update = ribUpdates.take();
                     switch (update.getOperation()) {
-                        case UPDATE:
-                            if (validateUpdate(update)) {
-                                processRibAdd(update);
-                            } else {
-                                log.debug("Rib UPDATE out of order: {} via {}",
-                                        update.getPrefix(), update.getRibEntry().getNextHop());
-                            }
-                            break;
-                        case DELETE:
-                            if (validateUpdate(update)) {
-                                processRibDelete(update);
-                            } else {
-                                log.debug("Rib DELETE out of order: {} via {}",
-                                        update.getPrefix(), update.getRibEntry().getNextHop());
-                            }
-                            break;
-                        default:
-                            log.error("Unknown operation {}", update.getOperation());
-                            break;
+                    case UPDATE:
+                        if (validateUpdate(update)) {
+                            processRibAdd(update);
+                        } else {
+                            log.debug("Rib UPDATE out of order: {} via {}",
+                                    update.getPrefix(), update.getRibEntry().getNextHop());
+                        }
+                        break;
+                    case DELETE:
+                        if (validateUpdate(update)) {
+                            processRibDelete(update);
+                        } else {
+                            log.debug("Rib DELETE out of order: {} via {}",
+                                    update.getPrefix(), update.getRibEntry().getNextHop());
+                        }
+                        break;
+                    default:
+                        log.error("Unknown operation {}", update.getOperation());
+                        break;
                     }
                 } catch (InterruptedException e) {
                     log.debug("Interrupted while taking from updates queue", e);
@@ -1368,10 +1383,11 @@
         RibEntry oldEntry = bgpRoutes.getValueForExactKey(
                 update.getPrefix().toBinaryString());
 
-        //If there is no existing entry we must assume this is the most recent
-        //update. However this might not always be the case as we might have a
-        //POST then DELETE reordering.
-        //if (oldEntry == null || !newEntry.getNextHop().equals(oldEntry.getNextHop())) {
+        // If there is no existing entry we must assume this is the most recent
+        // update. However this might not always be the case as we might have a
+        // POST then DELETE reordering.
+        // if (oldEntry == null ||
+        // !newEntry.getNextHop().equals(oldEntry.getNextHop())) {
         if (oldEntry == null) {
             return true;
         }
@@ -1387,7 +1403,7 @@
         }
 
         return newEntry.getSysUpTime() == oldEntry.getSysUpTime() &&
-               newEntry.getSequenceNum() > oldEntry.getSequenceNum();
+                newEntry.getSequenceNum() > oldEntry.getSequenceNum();
     }
 
     private Interface longestInterfacePrefixMatch(InetAddress address) {
@@ -1438,8 +1454,17 @@
     }
     */
 
+    // ******************
+    // IOFSwitchListener
+    // ******************
+
     @Override
-    public void addedSwitch(IOFSwitch sw) {
+    public void switchActivatedMaster(long swId) {
+        IOFSwitch sw = floodlightProvider.getSwitch(swId);
+        if (sw == null) {
+            log.warn("Added switch not available {} ", swId);
+            return;
+        }
         if (!topologyReady) {
             sw.clearAllFlowMods();
         }
@@ -1448,12 +1473,30 @@
     }
 
     @Override
-    public void removedSwitch(IOFSwitch sw) {
+    public void switchActivatedEqual(long swId) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void switchMasterToEqual(long swId) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void switchEqualToMaster(long swId) {
+        // for now treat as switchActivatedMaster
+        switchActivatedMaster(swId);
+    }
+
+    @Override
+    public void switchDisconnected(long swId) {
         // Not used
     }
 
     @Override
-    public void switchPortChanged(Long switchId) {
+    public void switchPortChanged(long swId, OFPortDesc port, PortChangeType pct) {
         // Not used
     }
 
diff --git a/src/main/java/net/onrc/onos/core/drivermanager/DriverManager.java b/src/main/java/net/onrc/onos/core/drivermanager/DriverManager.java
new file mode 100644
index 0000000..fe86077
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/drivermanager/DriverManager.java
@@ -0,0 +1,58 @@
+package net.onrc.onos.core.drivermanager;
+
+import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.internal.OFSwitchImplBase;
+
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A simple implementation of a driver manager that differentiates between
+ * connected switches using the OF Description Statistics Reply message.
+ */
+public final class DriverManager {
+
+    private static final Logger log = LoggerFactory.getLogger(DriverManager.class);
+
+    /**
+     * Return an IOFSwitch object based on switch's manufacturer description
+     * from OFDescStatsReply.
+     *
+     * @param desc DescriptionStatistics reply from the switch
+     * @return A IOFSwitch instance if the driver found an implementation for
+     *         the given description. Otherwise it returns OFSwitchImplBase
+     */
+    public static IOFSwitch getOFSwitchImpl(OFDescStatsReply desc, OFVersion ofv) {
+        String vendor = desc.getMfrDesc();
+        String hw = desc.getHwDesc();
+        if (vendor.startsWith("Stanford University, Ericsson Research and CPqD Research")
+                &&
+                hw.startsWith("OpenFlow 1.3 Reference Userspace Switch")) {
+            return new OFSwitchImplCPqD13(desc);
+        }
+
+        if (vendor.startsWith("Nicira") &&
+                hw.startsWith("Open vSwitch")) {
+            if (ofv == OFVersion.OF_10) {
+                return new OFSwitchImplOVS10(desc);
+            } else if (ofv == OFVersion.OF_13) {
+                return new OFSwitchImplOVS13(desc);
+            }
+        }
+
+        log.warn("DriverManager could not identify switch desc: {}. "
+                + "Assigning OFSwitchImplBase", desc);
+        OFSwitchImplBase base = new OFSwitchImplBase();
+        base.setSwitchDescription(desc);
+        // XXX S must set counter here - unidentified switch
+        return base;
+    }
+
+    /**
+     * Private constructor to avoid instantiation.
+     */
+    private DriverManager() {
+    }
+}
diff --git a/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplCPqD13.java b/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplCPqD13.java
new file mode 100644
index 0000000..c2bb344
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplCPqD13.java
@@ -0,0 +1,1208 @@
+package net.onrc.onos.core.drivermanager;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import net.floodlightcontroller.core.IFloodlightProviderService.Role;
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeAlreadyStarted;
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeCompleted;
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeNotStarted;
+import net.floodlightcontroller.core.internal.OFSwitchImplBase;
+
+import org.projectfloodlight.openflow.protocol.OFAsyncGetReply;
+import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
+import org.projectfloodlight.openflow.protocol.OFBucket;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFErrorMsg;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFGroupDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFGroupFeaturesStatsReply;
+import org.projectfloodlight.openflow.protocol.OFGroupType;
+import org.projectfloodlight.openflow.protocol.OFMatchV3;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFOxmList;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFStatsReply;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.protocol.instruction.OFInstruction;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmEthDst;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmEthSrc;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmEthType;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmInPort;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmIpv4DstMasked;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmMetadataMasked;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmMplsLabel;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmVlanVid;
+import org.projectfloodlight.openflow.types.EthType;
+import org.projectfloodlight.openflow.types.IPv4Address;
+import org.projectfloodlight.openflow.types.MacAddress;
+import org.projectfloodlight.openflow.types.OFBufferId;
+import org.projectfloodlight.openflow.types.OFGroup;
+import org.projectfloodlight.openflow.types.OFMetadata;
+import org.projectfloodlight.openflow.types.OFPort;
+import org.projectfloodlight.openflow.types.OFVlanVidMatch;
+import org.projectfloodlight.openflow.types.TableId;
+import org.projectfloodlight.openflow.types.U32;
+import org.projectfloodlight.openflow.types.U64;
+import org.projectfloodlight.openflow.util.HexString;
+
+/**
+ * OFDescriptionStatistics Vendor (Manufacturer Desc.): Stanford University,
+ * Ericsson Research and CPqD Research. Make (Hardware Desc.) : OpenFlow 1.3
+ * Reference Userspace Switch Model (Datapath Desc.) : None Software : Serial :
+ * None
+ */
+public class OFSwitchImplCPqD13 extends OFSwitchImplBase {
+    private static final int VLAN_ID_OFFSET = 16;
+    private AtomicBoolean driverHandshakeComplete;
+    private OFFactory factory;
+    private static final int OFPCML_NO_BUFFER = 0xffff;
+    // Configuration of asynch messages to controller. We need different
+    // asynch messages depending on role-equal or role-master.
+    // We don't want to get anything if we are slave.
+    private static final long SET_FLOW_REMOVED_MASK_MASTER = 0xf;
+    private static final long SET_PACKET_IN_MASK_MASTER = 0x7;
+    private static final long SET_PORT_STATUS_MASK_MASTER = 0x7;
+    private static final long SET_FLOW_REMOVED_MASK_EQUAL = 0x0;
+    private static final long SET_PACKET_IN_MASK_EQUAL = 0x0;
+    private static final long SET_PORT_STATUS_MASK_EQUAL = 0x7;
+    private static final long SET_ALL_SLAVE = 0x0;
+
+    private static final long TEST_FLOW_REMOVED_MASK = 0xf;
+    private static final long TEST_PACKET_IN_MASK = 0x7;
+    private static final long TEST_PORT_STATUS_MASK = 0x7;
+    private long barrierXidToWaitFor = -1;
+
+    private static final int TABLE_VLAN = 0;
+    private static final int TABLE_TMAC = 1;
+    private static final int TABLE_IPV4_UNICAST = 2;
+    private static final int TABLE_MPLS = 3;
+    private static final int TABLE_META = 4;
+    private static final int TABLE_ACL = 5;
+
+    private static final short MAX_PRIORITY = (short) 0xffff;
+    private static final short SLASH_24_PRIORITY = (short) 0xfff0;
+    private static final short SLASH_16_PRIORITY = (short) 0xff00;
+    private static final short SLASH_8_PRIORITY = (short) 0xf000;
+    private static final short MIN_PRIORITY = 0x0;
+    private static final U64 METADATA_MASK = U64.of(Long.MAX_VALUE << 1 | 0x1);
+
+    ConcurrentHashMap<Integer, OFGroup> l2groups;
+
+    public OFSwitchImplCPqD13(OFDescStatsReply desc) {
+        super();
+        driverHandshakeComplete = new AtomicBoolean(false);
+        l2groups = new ConcurrentHashMap<Integer, OFGroup>();
+        setSwitchDescription(desc);
+
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        return "OFSwitchImplCPqD13 [" + ((channel != null)
+                ? channel.getRemoteAddress() : "?")
+                + " DPID[" + ((stringId != null) ? stringId : "?") + "]]";
+    }
+
+    @Override
+    public void startDriverHandshake() throws IOException {
+        log.debug("Starting driver handshake for sw {}", getStringId());
+        if (startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeAlreadyStarted();
+        }
+        startDriverHandshakeCalled = true;
+        factory = floodlightProvider.getOFMessageFactory_13();
+        // configureSwitch();
+        sendBarrier(true);
+    }
+
+    @Override
+    public boolean isDriverHandshakeComplete() {
+        if (!startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeNotStarted();
+        }
+        return driverHandshakeComplete.get();
+    }
+
+    @Override
+    public void processDriverHandshakeMessage(OFMessage m) {
+        if (!startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeNotStarted();
+        }
+        if (driverHandshakeComplete.get()) {
+            throw new SwitchDriverSubHandshakeCompleted(m);
+        }
+
+        if (!startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeNotStarted();
+        }
+        if (driverHandshakeComplete.get()) {
+            throw new SwitchDriverSubHandshakeCompleted(m);
+        }
+
+        switch (m.getType()) {
+        case BARRIER_REPLY:
+            if (m.getXid() == barrierXidToWaitFor) {
+                driverHandshakeComplete.set(true);
+            }
+            break;
+
+        case ERROR:
+            log.error("Switch {} Error {}", getStringId(), (OFErrorMsg) m);
+            break;
+
+        case FEATURES_REPLY:
+            break;
+        case FLOW_REMOVED:
+            break;
+        case GET_ASYNC_REPLY:
+            OFAsyncGetReply asrep = (OFAsyncGetReply) m;
+            decodeAsyncGetReply(asrep);
+            break;
+
+        case PACKET_IN:
+            break;
+        case PORT_STATUS:
+            break;
+        case QUEUE_GET_CONFIG_REPLY:
+            break;
+        case ROLE_REPLY:
+            break;
+
+        case STATS_REPLY:
+            processStatsReply((OFStatsReply) m);
+            break;
+
+        default:
+            log.debug("Received message {} during switch-driver subhandshake "
+                    + "from switch {} ... Ignoring message", m, getStringId());
+
+        }
+    }
+
+    private void configureSwitch() throws IOException {
+        // setAsyncConfig();
+        // getTableFeatures();
+        sendGroupFeaturesRequest();
+        setL2Groups();
+        sendBarrier(false);
+        setL3Groups();
+        setL25Groups();
+        sendGroupDescRequest();
+        populateTableVlan();
+        populateTableTMac();
+        populateIpTable();
+        populateMplsTable();
+        populateTableMissEntry(TABLE_ACL, false, false, false, -1);
+        sendBarrier(true);
+    }
+
+    private void setAsyncConfig() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>(3);
+        OFMessage setAC = null;
+
+        if (role == Role.MASTER) {
+            setAC = factory.buildAsyncSet()
+                    .setFlowRemovedMaskEqualMaster(SET_FLOW_REMOVED_MASK_MASTER)
+                    .setPacketInMaskEqualMaster(SET_PACKET_IN_MASK_MASTER)
+                    .setPortStatusMaskEqualMaster(SET_PORT_STATUS_MASK_MASTER)
+                    .setFlowRemovedMaskSlave(SET_ALL_SLAVE)
+                    .setPacketInMaskSlave(SET_ALL_SLAVE)
+                    .setPortStatusMaskSlave(SET_ALL_SLAVE)
+                    .setXid(getNextTransactionId())
+                    .build();
+        } else if (role == Role.EQUAL) {
+            setAC = factory.buildAsyncSet()
+                    .setFlowRemovedMaskEqualMaster(SET_FLOW_REMOVED_MASK_EQUAL)
+                    .setPacketInMaskEqualMaster(SET_PACKET_IN_MASK_EQUAL)
+                    .setPortStatusMaskEqualMaster(SET_PORT_STATUS_MASK_EQUAL)
+                    .setFlowRemovedMaskSlave(SET_ALL_SLAVE)
+                    .setPacketInMaskSlave(SET_ALL_SLAVE)
+                    .setPortStatusMaskSlave(SET_ALL_SLAVE)
+                    .setXid(getNextTransactionId())
+                    .build();
+        }
+        msglist.add(setAC);
+
+        OFMessage br = factory.buildBarrierRequest()
+                .setXid(getNextTransactionId())
+                .build();
+        msglist.add(br);
+
+        OFMessage getAC = factory.buildAsyncGetRequest()
+                .setXid(getNextTransactionId())
+                .build();
+        msglist.add(getAC);
+
+        write(msglist);
+    }
+
+    private void decodeAsyncGetReply(OFAsyncGetReply rep) {
+        long frm = rep.getFlowRemovedMaskEqualMaster();
+        //long frs = rep.getFlowRemovedMaskSlave();
+        long pim = rep.getPacketInMaskEqualMaster();
+        //long pis = rep.getPacketInMaskSlave();
+        long psm = rep.getPortStatusMaskEqualMaster();
+        //long pss = rep.getPortStatusMaskSlave();
+
+        if (role == Role.MASTER || role == Role.EQUAL) { // should separate
+            log.info("FRM:{}", HexString.toHexString((frm & TEST_FLOW_REMOVED_MASK)));
+            log.info("PIM:{}", HexString.toHexString((pim & TEST_PACKET_IN_MASK)));
+            log.info("PSM:{}", HexString.toHexString((psm & TEST_PORT_STATUS_MASK)));
+        }
+
+    }
+
+    private void getTableFeatures() throws IOException {
+        OFMessage gtf = factory.buildTableFeaturesStatsRequest()
+                .setXid(getNextTransactionId())
+                .build();
+        write(gtf, null);
+    }
+
+    private void sendGroupFeaturesRequest() throws IOException {
+        OFMessage gfr = factory.buildGroupFeaturesStatsRequest()
+                .setXid(getNextTransactionId())
+                .build();
+        write(gfr, null);
+    }
+
+    private void sendGroupDescRequest() throws IOException {
+        OFMessage gdr = factory.buildGroupDescStatsRequest()
+                .setXid(getNextTransactionId())
+                .build();
+        write(gdr, null);
+    }
+
+    /*Create L2 interface groups for all physical ports
+     Naming convention followed is the same as OF-DPA spec
+     eg. port 1 with allowed vlan 10, is enveloped in group with id,
+         0x0 00a 0001, where the uppermost 4 bits identify an L2 interface,
+         the next 12 bits identify the vlan-id, and the lowermost 16 bits
+         identify the port number.*/
+    private void setL2Groups() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        for (OFPortDesc p : getPorts()) {
+            int pnum = p.getPortNo().getPortNumber();
+            int portVlan = getVlanConfig(pnum);
+            if (U32.of(pnum).compareTo(U32.of(OFPort.MAX.getPortNumber())) < 1) {
+                OFGroup gl2 = OFGroup.of(pnum | (portVlan << VLAN_ID_OFFSET));
+                OFAction out = factory.actions().buildOutput()
+                        .setPort(p.getPortNo()).build();
+                OFAction popVlan = factory.actions().popVlan();
+                List<OFAction> actions = new ArrayList<OFAction>();
+                actions.add(popVlan);
+                actions.add(out);
+                OFBucket bucket = factory.buildBucket()
+                        .setActions(actions).build();
+                List<OFBucket> buckets = Collections.singletonList(bucket);
+                OFMessage gmAdd = factory.buildGroupAdd()
+                        .setGroup(gl2)
+                        .setBuckets(buckets)
+                        .setGroupType(OFGroupType.INDIRECT)
+                        .setXid(getNextTransactionId())
+                        .build();
+                msglist.add(gmAdd);
+                l2groups.put(pnum, gl2);
+            }
+        }
+        log.debug("Creating {} L2 groups in sw {}", msglist.size(), getStringId());
+        write(msglist);
+    }
+
+    private int getVlanConfig(int portnum) {
+        int portVlan = 10 * portnum;
+        if ((getId() == 0x1 && portnum == 6) ||
+                (getId() == 0x2) ||
+                (getId() == 0x3 && portnum == 2)) {
+            portVlan = 192; // 0xc0
+        }
+        return portVlan;
+    }
+
+    private MacAddress getRouterMacAddr() {
+        if (getId() == 0x3) {
+            return MacAddress.of("00:00:07:07:07:80"); // router mac
+        }
+        if (getId() == 0x1) {
+            return MacAddress.of("00:00:01:01:01:80");
+        }
+        // switch 0x2
+        return MacAddress.of("00:00:02:02:02:80");
+    }
+
+    // only for ports connected to other routers
+    private OFAction getDestAction(int portnum) {
+        OFAction setDA = null;
+        MacAddress dAddr = null;
+        if (getId() == 0x1 && portnum == 6) { // connected to switch 2
+            dAddr = MacAddress.of("00:00:02:02:02:80");
+        }
+        if (getId() == 0x2) {
+            if (portnum == 1) { // connected to sw 1
+                dAddr = MacAddress.of("00:00:01:01:01:80");
+            } else if (portnum == 2) { // connected to sw 3
+                dAddr = MacAddress.of("00:00:07:07:07:80");
+            }
+        }
+        if (getId() == 0x3) {
+            if (portnum == 2) { // connected to switch 2
+                dAddr = MacAddress.of("00:00:02:02:02:80");
+            }
+        }
+
+        if (dAddr != null) {
+            OFOxmEthDst dstAddr = factory.oxms().ethDst(dAddr);
+            setDA = factory.actions().buildSetField()
+                    .setField(dstAddr).build();
+        }
+        return setDA;
+    }
+
+    /*
+     * L3 groups are created for all router ports and they all point to corresponding
+     * L2 groups. Only the ports that connect to other routers will have the
+     * DA set.
+     */
+    private void setL3Groups() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        for (OFGroup gl2 : l2groups.values()) {
+            int gnum = gl2.getGroupNumber();
+            int portnum = gnum & 0x0000ffff;
+            int vlanid = ((gnum & 0x0fff0000) >> VLAN_ID_OFFSET);
+            MacAddress sAddr = getRouterMacAddr();
+
+            OFGroup gl3 = OFGroup.of(0x20000000 | portnum);
+            OFAction group = factory.actions().buildGroup()
+                    .setGroup(gl2).build();
+            OFOxmEthSrc srcAddr = factory.oxms().ethSrc(sAddr);
+            OFAction setSA = factory.actions().buildSetField()
+                    .setField(srcAddr).build();
+            OFOxmVlanVid vid = factory.oxms().vlanVid(OFVlanVidMatch.ofVlan(vlanid));
+            OFAction setVlan = factory.actions().buildSetField()
+                    .setField(vid).build();
+            OFAction decTtl = factory.actions().decNwTtl();
+
+            List<OFAction> actions = new ArrayList<OFAction>();
+            actions.add(decTtl); // decrement the IP TTL/do-checksum/check TTL
+                                 // and MTU
+            actions.add(setVlan); // set the vlan-id of the exit-port (and
+                                  // l2group)
+            actions.add(setSA); // set this routers mac address
+            // make L3Unicast group setDA for known (configured) ports
+            // that connect to other routers
+            OFAction setDA = getDestAction(portnum);
+            if (setDA != null) {
+                actions.add(setDA);
+            }
+            actions.add(group);
+
+            OFBucket bucket = factory.buildBucket()
+                    .setActions(actions).build();
+            List<OFBucket> buckets = Collections.singletonList(bucket);
+            OFMessage gmAdd = factory.buildGroupAdd()
+                    .setGroup(gl3)
+                    .setBuckets(buckets)
+                    .setGroupType(OFGroupType.INDIRECT)
+                    .setXid(getNextTransactionId())
+                    .build();
+            msglist.add(gmAdd);
+        }
+        write(msglist);
+        log.debug("Creating {} L3 groups in sw {}", msglist.size(), getStringId());
+    }
+
+    /*
+     * L2.5 or mpls-unicast groups are only created for those router ports
+     * connected to other router ports. They differ from the corresponding
+     * L3-unicast group only by the fact that they decrement the MPLS TTL
+     * instead of the IP ttl
+     */
+    private void setL25Groups() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        for (OFGroup gl2 : l2groups.values()) {
+            int gnum = gl2.getGroupNumber();
+            int portnum = gnum & 0x0000ffff;
+            int vlanid = ((gnum & 0x0fff0000) >> VLAN_ID_OFFSET);
+            MacAddress sAddr = getRouterMacAddr();
+            OFAction setDA = getDestAction(portnum);
+            // setDA will only be non-null for ports connected to routers
+            if (setDA != null) {
+                OFGroup gl3 = OFGroup.of(0xa0000000 | portnum); // different id
+                                                                // for mpls
+                                                                // group
+                OFAction group = factory.actions().buildGroup()
+                        .setGroup(gl2).build();
+                OFOxmEthSrc srcAddr = factory.oxms().ethSrc(sAddr);
+                OFAction setSA = factory.actions().buildSetField()
+                        .setField(srcAddr).build();
+                OFOxmVlanVid vid = factory.oxms().vlanVid(OFVlanVidMatch.ofVlan(vlanid));
+                OFAction setVlan = factory.actions().buildSetField()
+                        .setField(vid).build();
+                OFAction decMplsTtl = factory.actions().decMplsTtl();
+                List<OFAction> actions = new ArrayList<OFAction>();
+                actions.add(decMplsTtl); // decrement the MPLS
+                                         // TTL/do-checksum/check TTL and MTU
+                actions.add(setVlan); // set the vlan-id of the exit-port (and
+                                      // l2group)
+                actions.add(setSA); // set this routers mac address
+                actions.add(setDA);
+                actions.add(group);
+                OFBucket bucket = factory.buildBucket()
+                        .setActions(actions).build();
+                List<OFBucket> buckets = Collections.singletonList(bucket);
+                OFMessage gmAdd = factory.buildGroupAdd()
+                        .setGroup(gl3)
+                        .setBuckets(buckets)
+                        .setGroupType(OFGroupType.INDIRECT)
+                        .setXid(getNextTransactionId())
+                        .build();
+                msglist.add(gmAdd);
+            }
+        }
+        write(msglist);
+        log.debug("Creating {} MPLS groups in sw {}", msglist.size(), getStringId());
+    }
+
+    /* Using ECMP groups
+     *
+     * OFGroup group47 = OFGroup.of(47);
+        OFAction outgroup1 = factory.actions()
+                        .buildGroup()
+                        .setGroup(group61)
+                        .build();
+        OFBucket buc47_1 = factory.buildBucket()
+                        .setWeight(1)
+                        .setActions(Collections.singletonList(outgroup1))
+                        .build();
+        OFAction outgroup2 = factory.actions()
+                        .buildGroup()
+                        .setGroup(group62)
+                        .build();
+        OFBucket buc47_2 = factory.buildBucket()
+                        .setWeight(1)
+                        .setActions(Collections.singletonList(outgroup2))
+                        .build();
+        List<OFBucket> buckets47 = new ArrayList<OFBucket>();
+        buckets47.add(buc47_1);
+        buckets47.add(buc47_2);
+        OFMessage gmS12 = factory.buildGroupAdd()
+                        .setGroup(group47)
+                        .setBuckets(buckets47)
+                        .setGroupType(OFGroupType.SELECT)
+                        .setXid(getNextTransactionId())
+                        .build();
+        write(gmS12, null);     */
+
+    private void processStatsReply(OFStatsReply sr) {
+        switch (sr.getStatsType()) {
+        case AGGREGATE:
+            break;
+        case DESC:
+            break;
+        case EXPERIMENTER:
+            break;
+        case FLOW:
+            break;
+        case GROUP_DESC:
+            processGroupDesc((OFGroupDescStatsReply) sr);
+            break;
+        case GROUP_FEATURES:
+            processGroupFeatures((OFGroupFeaturesStatsReply) sr);
+            break;
+        case METER_CONFIG:
+            break;
+        case METER_FEATURES:
+            break;
+        case PORT_DESC:
+            break;
+        case TABLE_FEATURES:
+            break;
+        default:
+            break;
+
+        }
+    }
+
+    private void processGroupFeatures(OFGroupFeaturesStatsReply gfsr) {
+        log.info("Sw: {} Group Features {}", getStringId(), gfsr);
+    }
+
+    private void processGroupDesc(OFGroupDescStatsReply gdsr) {
+        log.info("Sw: {} Group Desc {}", getStringId(), gdsr);
+    }
+
+    private void populateTableVlan() throws IOException {
+        // for all incoming ports assign configured port-vlans
+        // currently assign portnum*10 -> vlanid to access ports
+        // and vlan 192 to router to router ports
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        for (OFPortDesc p : getPorts()) {
+            int pnum = p.getPortNo().getPortNumber();
+            if (U32.of(pnum).compareTo(U32.of(OFPort.MAX.getPortNumber())) < 1) {
+                int vlanid = getVlanConfig(pnum);
+                OFOxmInPort oxp = factory.oxms().inPort(p.getPortNo());
+                OFOxmVlanVid oxv = factory.oxms()
+                        .vlanVid(OFVlanVidMatch.UNTAGGED);
+                OFOxmList oxmList = OFOxmList.of(oxp, oxv);
+                OFMatchV3 match = factory.buildMatchV3()
+                        .setOxmList(oxmList).build();
+                OFOxmVlanVid vidToSet = factory.oxms()
+                        .vlanVid(OFVlanVidMatch.ofVlan(vlanid));
+                OFAction pushVlan = factory.actions().pushVlan(EthType.VLAN_FRAME);
+                OFAction setVlan = factory.actions().setField(vidToSet);
+                List<OFAction> actionlist = new ArrayList<OFAction>();
+                actionlist.add(pushVlan);
+                actionlist.add(setVlan);
+                OFInstruction appAction = factory.instructions().buildApplyActions()
+                        .setActions(actionlist).build();
+                OFInstruction gotoTbl = factory.instructions().buildGotoTable()
+                        .setTableId(TableId.of(TABLE_TMAC)).build();
+                List<OFInstruction> instructions = new ArrayList<OFInstruction>();
+                instructions.add(appAction);
+                instructions.add(gotoTbl);
+                OFMessage flowEntry = factory.buildFlowAdd()
+                        .setTableId(TableId.of(TABLE_VLAN))
+                        .setMatch(match)
+                        .setInstructions(instructions)
+                        .setPriority(1000) // does not matter - all rules
+                                           // exclusive
+                        .setBufferId(OFBufferId.NO_BUFFER)
+                        .setIdleTimeout(0)
+                        .setHardTimeout(0)
+                        .setXid(getNextTransactionId())
+                        .build();
+                msglist.add(flowEntry);
+            }
+        }
+        // table-vlan has no table-miss entry, and so packets that miss are
+        // essentially dropped
+        write(msglist);
+        log.debug("Adding {} vlan-rules in sw {}", msglist.size(), getStringId());
+    }
+
+    private void populateTableTMac() throws IOException {
+        // match for ip packets
+        OFOxmEthType oxe = factory.oxms().ethType(EthType.IPv4);
+        OFOxmList oxmListIp = OFOxmList.of(oxe);
+        OFMatchV3 matchIp = factory.buildMatchV3()
+                .setOxmList(oxmListIp).build();
+        OFInstruction gotoTblIp = factory.instructions().buildGotoTable()
+                .setTableId(TableId.of(TABLE_IPV4_UNICAST)).build();
+        List<OFInstruction> instructionsIp = Collections.singletonList(gotoTblIp);
+        OFMessage ipEntry = factory.buildFlowAdd()
+                .setTableId(TableId.of(TABLE_TMAC))
+                .setMatch(matchIp)
+                .setInstructions(instructionsIp)
+                .setPriority(1000) // strict priority required lower than
+                                   // multicastMac
+                .setBufferId(OFBufferId.NO_BUFFER)
+                .setIdleTimeout(0)
+                .setHardTimeout(0)
+                .setXid(getNextTransactionId())
+                .build();
+
+        // match for mpls packets
+        OFOxmEthType oxmpls = factory.oxms().ethType(EthType.MPLS_UNICAST);
+        OFOxmList oxmListMpls = OFOxmList.of(oxmpls);
+        OFMatchV3 matchMpls = factory.buildMatchV3()
+                .setOxmList(oxmListMpls).build();
+        OFInstruction gotoTblMpls = factory.instructions().buildGotoTable()
+                .setTableId(TableId.of(TABLE_MPLS)).build();
+        List<OFInstruction> instructionsMpls = Collections.singletonList(gotoTblMpls);
+        OFMessage mplsEntry = factory.buildFlowAdd()
+                .setTableId(TableId.of(TABLE_TMAC))
+                .setMatch(matchMpls)
+                .setInstructions(instructionsMpls)
+                .setPriority(1001) // strict priority required lower than
+                                   // multicastMac
+                .setBufferId(OFBufferId.NO_BUFFER)
+                .setIdleTimeout(0)
+                .setHardTimeout(0)
+                .setXid(getNextTransactionId())
+                .build();
+
+        // match for everything else to send to controller. Essentially
+        // the table miss flow entry
+        populateTableMissEntry(TABLE_TMAC, true, false, false, -1);
+        log.debug("Adding termination-mac-rules in sw {}", getStringId());
+        List<OFMessage> msglist = new ArrayList<OFMessage>(2);
+        msglist.add(ipEntry);
+        msglist.add(mplsEntry);
+        write(msglist);
+    }
+
+    private List<String> getMyIps() { // send to controller
+        List<String> myIps = new ArrayList<String>();
+        if (getId() == 0x1) {
+            myIps.add("10.0.2.128");
+            myIps.add("10.0.3.128");
+            myIps.add("10.0.1.128");
+            myIps.add("192.168.0.1");
+        }
+        if (getId() == 0x2) {
+            myIps.add("192.168.0.2");
+        }
+        if (getId() == 0x3) {
+            myIps.add("192.168.0.3");
+            myIps.add("7.7.7.128");
+        }
+        return myIps;
+    }
+
+    private List<String> getMySubnetIps() { // send to controller
+        List<String> subnetIps = new ArrayList<String>();
+        if (getId() == 0x1) {
+            subnetIps.add("10.0.2.0");
+            subnetIps.add("10.0.3.0");
+            subnetIps.add("10.0.1.0");
+        }
+        // TODO needed?
+        //if (getId() == 0x2) {
+        //}
+        if (getId() == 0x3) {
+            subnetIps.add("7.7.7.0");
+        }
+        return subnetIps;
+    }
+
+    private static class RouteEntry {
+        String prefix;
+        String mask;
+        int nextHopPort;
+        String dstMac;
+        int label;
+
+        public RouteEntry(String prefix, String mask, int nextHopPort, int label) {
+            this.prefix = prefix;
+            this.mask = mask;
+            this.nextHopPort = nextHopPort;
+            this.label = label;
+        }
+
+        public RouteEntry(String prefix, int nextHopPort, String dstMac) {
+            this.prefix = prefix;
+            this.nextHopPort = nextHopPort;
+            this.dstMac = dstMac;
+        }
+    }
+
+    // send out of mpls-group where the next-hop mac-da is already set
+    private List<RouteEntry> getRouterNextHopIps() {
+        List<RouteEntry> routerNextHopIps = new ArrayList<RouteEntry>();
+        if (getId() == 0x1) {
+            routerNextHopIps
+                    .add(new RouteEntry("192.168.0.2", "255.255.255.255", 6, 102));
+            routerNextHopIps
+                    .add(new RouteEntry("192.168.0.3", "255.255.255.255", 6, 103));
+            routerNextHopIps.add(new RouteEntry("7.7.7.0", "255.255.255.0", 6, 103));
+        }
+        //if (getId() == 0x2) {
+            /* These are required for normal IP routing without labels.
+            routerNextHopIps.add(new RouteEntry("192.168.0.1","255.255.255.255",1));
+            routerNextHopIps.add(new RouteEntry("192.168.0.3","255.255.255.255",2));
+            routerNextHopIps.add(new RouteEntry("10.0.1.0","255.255.255.0",1));
+            routerNextHopIps.add(new RouteEntry("10.0.2.0","255.255.255.0",1));
+            routerNextHopIps.add(new RouteEntry("10.0.3.0","255.255.255.0",1));
+            routerNextHopIps.add(new RouteEntry("7.7.7.0","255.255.255.0",2));*/
+        //}
+        if (getId() == 0x3) {
+            routerNextHopIps
+                    .add(new RouteEntry("192.168.0.2", "255.255.255.255", 2, 102));
+            routerNextHopIps
+                    .add(new RouteEntry("192.168.0.1", "255.255.255.255", 2, 101));
+            routerNextHopIps.add(new RouteEntry("10.0.1.0", "255.255.255.0", 2, 101));
+            routerNextHopIps.add(new RouteEntry("10.0.2.0", "255.255.255.0", 2, 101));
+            routerNextHopIps.add(new RouteEntry("10.0.3.0", "255.255.255.0", 2, 101));
+        }
+        return routerNextHopIps;
+    }
+
+    // known host mac-addr, setDA/send out of l3group
+    private List<RouteEntry> getHostNextHopIps() {
+        List<RouteEntry> hostNextHopIps = new ArrayList<RouteEntry>();
+        if (getId() == 0x1) {
+            hostNextHopIps.add(new RouteEntry("10.0.2.1", 4, "00:00:00:00:02:01"));
+            hostNextHopIps.add(new RouteEntry("10.0.3.1", 5, "00:00:00:00:03:01"));
+        }
+        // TODO needed?
+        //if (getId() == 0x2) {
+        //}
+        if (getId() == 0x3) {
+            hostNextHopIps.add(new RouteEntry("7.7.7.7", 1, "00:00:07:07:07:07"));
+        }
+        return hostNextHopIps;
+    }
+
+    private void populateIpTable() throws IOException {
+        populateMyIps();
+        populateMySubnets();
+        populateRoutes();
+        populateHostRoutes();
+
+        // match for everything else to send to ACL table. Essentially
+        // the table miss flow entry
+        populateTableMissEntry(TABLE_IPV4_UNICAST, false, true,
+                true, TABLE_ACL);
+    }
+
+    private void populateMyIps() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        // first all my ip's as exact-matches
+        // write-action instruction to send to controller
+        List<String> myIps = getMyIps();
+        for (int i = 0; i < myIps.size(); i++) {
+            OFOxmEthType ethTypeIp = factory.oxms()
+                    .ethType(EthType.IPv4);
+            OFOxmIpv4DstMasked ipPrefix = factory.oxms()
+                    .ipv4DstMasked(IPv4Address.of(myIps.get(i)), IPv4Address.NO_MASK);
+            OFOxmList oxmListSlash32 = OFOxmList.of(ethTypeIp, ipPrefix);
+            OFMatchV3 match = factory.buildMatchV3()
+                    .setOxmList(oxmListSlash32).build();
+            OFAction outc = factory.actions().buildOutput()
+                    .setPort(OFPort.CONTROLLER).setMaxLen(OFPCML_NO_BUFFER)
+                    .build();
+            OFInstruction writeInstr = factory.instructions().buildWriteActions()
+                    .setActions(Collections.singletonList(outc)).build();
+            OFInstruction gotoInstr = factory.instructions().buildGotoTable()
+                    .setTableId(TableId.of(TABLE_ACL)).build();
+            List<OFInstruction> instructions = new ArrayList<OFInstruction>();
+            instructions.add(writeInstr);
+            instructions.add(gotoInstr);
+            OFMessage myIpEntry = factory.buildFlowAdd()
+                    .setTableId(TableId.of(TABLE_IPV4_UNICAST))
+                    .setMatch(match)
+                    .setInstructions(instructions)
+                    .setPriority(MAX_PRIORITY) // highest priority for exact
+                                               // match
+                    .setBufferId(OFBufferId.NO_BUFFER)
+                    .setIdleTimeout(0)
+                    .setHardTimeout(0)
+                    .setXid(getNextTransactionId())
+                    .build();
+            msglist.add(myIpEntry);
+        }
+        write(msglist);
+        log.debug("Adding {} my-ip-rules in sw {}", msglist.size(), getStringId());
+    }
+
+    private void populateMySubnets() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        // next prefix-based subnet-IP's configured on my interfaces
+        // need to ARP for exact-IP, so write-action instruction to send to
+        // controller
+        // this has different mask and priority than earlier case
+        List<String> subnetIps = getMySubnetIps();
+        for (int i = 0; i < subnetIps.size(); i++) {
+            OFOxmEthType ethTypeIp = factory.oxms()
+                    .ethType(EthType.IPv4);
+            OFOxmIpv4DstMasked ipPrefix = factory.oxms().ipv4DstMasked(
+                    IPv4Address.of(subnetIps.get(i)),
+                    IPv4Address.of(0xffffff00)); // '/24' mask
+            OFOxmList oxmListSlash24 = OFOxmList.of(ethTypeIp, ipPrefix);
+            OFMatchV3 match = factory.buildMatchV3()
+                    .setOxmList(oxmListSlash24).build();
+            OFAction outc = factory.actions().buildOutput()
+                    .setPort(OFPort.CONTROLLER).setMaxLen(OFPCML_NO_BUFFER)
+                    .build();
+            OFInstruction writeInstr = factory.instructions().buildWriteActions()
+                    .setActions(Collections.singletonList(outc)).build();
+            OFInstruction gotoInstr = factory.instructions().buildGotoTable()
+                    .setTableId(TableId.of(TABLE_ACL)).build();
+            List<OFInstruction> instructions = new ArrayList<OFInstruction>();
+            instructions.add(writeInstr);
+            instructions.add(gotoInstr);
+            OFMessage myIpEntry = factory.buildFlowAdd()
+                    .setTableId(TableId.of(TABLE_IPV4_UNICAST))
+                    .setMatch(match)
+                    .setInstructions(instructions)
+                    .setPriority(SLASH_24_PRIORITY)
+                    .setBufferId(OFBufferId.NO_BUFFER)
+                    .setIdleTimeout(0)
+                    .setHardTimeout(0)
+                    .setXid(getNextTransactionId())
+                    .build();
+            msglist.add(myIpEntry);
+        }
+        write(msglist);
+        log.debug("Adding {} subnet-ip-rules in sw {}", msglist.size(), getStringId());
+        msglist.clear();
+    }
+
+    private void populateRoutes() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        // addresses where I know the next-hop's mac-address because it is a
+        // router port - so I have an L3 interface to it (and an MPLS interface)
+        List<RouteEntry> routerNextHopIps = getRouterNextHopIps();
+        for (int i = 0; i < routerNextHopIps.size(); i++) {
+            OFOxmEthType ethTypeIp = factory.oxms()
+                    .ethType(EthType.IPv4);
+            OFOxmIpv4DstMasked ipPrefix = factory.oxms()
+                    .ipv4DstMasked(
+                            IPv4Address.of(routerNextHopIps.get(i).prefix),
+                            IPv4Address.of(routerNextHopIps.get(i).mask)
+                    );
+            OFOxmList oxmListSlash32 = OFOxmList.of(ethTypeIp, ipPrefix);
+            OFMatchV3 match = factory.buildMatchV3()
+                    .setOxmList(oxmListSlash32).build();
+            OFAction outg = factory.actions().buildGroup()
+                    .setGroup(OFGroup.of(0xa0000000 | // mpls group id
+                            routerNextHopIps.get(i).nextHopPort))
+                    .build();
+            // lots of actions before forwarding to mpls group, and
+            // unfortunately
+            // they need to be apply-actions
+
+            OFAction pushlabel = factory.actions().pushMpls(EthType.MPLS_UNICAST);
+            OFOxmMplsLabel l = factory.oxms()
+                    .mplsLabel(U32.of(routerNextHopIps.get(i).label));
+            OFAction setlabelid = factory.actions().buildSetField()
+                    .setField(l).build();
+            OFAction copyTtlOut = factory.actions().copyTtlOut();
+            // OFAction setBos =
+            // factory.actions().buildSetField().setField(bos).build();
+
+            /*
+            writeActions.add(pushlabel);  // need to be apply actions so can be
+            writeActions.add(copyTtlOut); // matched in pseudo-table
+            //writeActions.add(setlabelid); // bad support in cpqd
+            //writeActions.add(setBos); no support in loxigen
+            */
+
+            List<OFAction> applyActions = new ArrayList<OFAction>();
+            applyActions.add(pushlabel);
+            applyActions.add(copyTtlOut);
+            OFInstruction applyInstr = factory.instructions().buildApplyActions()
+                    .setActions(applyActions).build();
+            List<OFAction> writeActions = new ArrayList<OFAction>();
+            writeActions.add(outg); // group will decr mpls-ttl, set mac-sa/da,
+                                    // vlan
+            OFInstruction writeInstr = factory.instructions().buildWriteActions()
+                    .setActions(writeActions).build();
+
+            // necessary to match in pseudo-table to overcome cpqd 1.3 flaw
+            OFInstruction writeMeta = factory.instructions().buildWriteMetadata()
+                    .setMetadata(U64.of(routerNextHopIps.get(i).label))
+                    .setMetadataMask(METADATA_MASK).build();
+            /*OFInstruction gotoInstr = factory.instructions().buildGotoTable()
+                        .setTableId(TableId.of(TABLE_ACL)).build();*/
+            OFInstruction gotoInstr = factory.instructions().buildGotoTable()
+                    .setTableId(TableId.of(TABLE_META)).build();
+            List<OFInstruction> instructions = new ArrayList<OFInstruction>();
+            instructions.add(applyInstr);
+            // instructions.add(writeInstr);// cannot write here - causes switch
+            // to crash
+            instructions.add(writeMeta);
+            instructions.add(gotoInstr);
+
+            int priority = -1;
+            if (routerNextHopIps.get(i).mask.equals("255.255.255.255")) {
+                priority = MAX_PRIORITY;
+            } else {
+                priority = SLASH_24_PRIORITY;
+            }
+            OFMessage myIpEntry = factory.buildFlowAdd()
+                    .setTableId(TableId.of(TABLE_IPV4_UNICAST))
+                    .setMatch(match)
+                    .setInstructions(instructions)
+                    .setPriority(priority)
+                    .setBufferId(OFBufferId.NO_BUFFER)
+                    .setIdleTimeout(0)
+                    .setHardTimeout(0)
+                    .setXid(getNextTransactionId())
+                    .build();
+            msglist.add(myIpEntry);
+
+            // need to also handle psuedo-table entries to match-metadata and
+            // set mpls
+            // label-id
+            OFOxmEthType ethTypeMpls = factory.oxms()
+                    .ethType(EthType.MPLS_UNICAST);
+            OFOxmMetadataMasked meta = factory.oxms()
+                    .metadataMasked(
+                            OFMetadata.ofRaw(routerNextHopIps.get(i).label),
+                            OFMetadata.NO_MASK);
+            OFOxmList oxmListMeta = OFOxmList.of(ethTypeMpls, meta);
+            OFMatchV3 matchMeta = factory.buildMatchV3()
+                    .setOxmList(oxmListMeta).build();
+            List<OFAction> writeActions2 = new ArrayList<OFAction>();
+            writeActions2.add(setlabelid);
+            OFAction outg2 = factory.actions().buildGroup()
+                    .setGroup(OFGroup.of(routerNextHopIps.get(i).nextHopPort |
+                            (192 << VLAN_ID_OFFSET)))
+                    .build();
+            writeActions2.add(outg2);
+            OFInstruction writeInstr2 = factory.instructions().buildWriteActions()
+                    .setActions(writeActions2).build();
+            OFInstruction gotoInstr2 = factory.instructions().buildGotoTable()
+                    .setTableId(TableId.of(TABLE_ACL)).build();
+            List<OFInstruction> instructions2 = new ArrayList<OFInstruction>();
+            // unfortunately have to apply this action too
+            OFInstruction applyInstr2 = factory.instructions().buildApplyActions()
+                    .setActions(writeActions2).build();
+            instructions2.add(applyInstr2);
+            // instructions2.add(writeInstr2);
+            // instructions2.add(gotoInstr2);
+
+            /*OFMatchV3 match3 = factory.buildMatchV3()
+                        .setOxmList(OFOxmList.of(meta)).build();
+            OFInstruction clearInstruction = factory.instructions().clearActions();
+            List<OFInstruction> instructions3 = new ArrayList<OFInstruction>();
+            OFAction outc = factory.actions().buildOutput()
+                        .setPort(OFPort.CONTROLLER).setMaxLen(OFPCML_NO_BUFFER)
+                        .build();
+            OFInstruction writec = factory.instructions()
+                        .writeActions(Collections.singletonList(outc));
+            instructions3.add(clearInstruction);
+            instructions3.add(writec);
+            instructions3.add(gotoInstr2); */
+            OFMessage myMetaEntry = factory.buildFlowAdd()
+                    .setTableId(TableId.of(TABLE_META))
+                    .setMatch(matchMeta)
+                    .setInstructions(instructions2)
+                    .setPriority(MAX_PRIORITY)
+                    .setBufferId(OFBufferId.NO_BUFFER)
+                    .setIdleTimeout(0)
+                    .setHardTimeout(0)
+                    .setXid(getNextTransactionId())
+                    .build();
+            msglist.add(myMetaEntry);
+
+        }
+        write(msglist);
+        log.debug("Adding {} next-hop-router-rules in sw {}", msglist.size(),
+                getStringId());
+
+        // add a table-miss entry to table 4 for debugging - leave it out
+        // unclear packet state - causes switch to crash
+        // populateTableMissEntry(TABLE_META, false, true,
+        // true, TABLE_ACL);
+    }
+
+    private void populateHostRoutes() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        // addresses where I know the next hop's mac-address and I can set the
+        // destination mac in the match-instruction.write-action
+        // either I sent out arp-request or I got an arp-request from this host
+        List<RouteEntry> hostNextHopIps = getHostNextHopIps();
+        for (int i = 0; i < hostNextHopIps.size(); i++) {
+            OFOxmEthType ethTypeIp = factory.oxms()
+                    .ethType(EthType.IPv4);
+            OFOxmIpv4DstMasked ipPrefix = factory.oxms()
+                    .ipv4DstMasked(
+                            IPv4Address.of(hostNextHopIps.get(i).prefix),
+                            IPv4Address.NO_MASK); // host addr should be /32
+            OFOxmList oxmListSlash32 = OFOxmList.of(ethTypeIp, ipPrefix);
+            OFMatchV3 match = factory.buildMatchV3()
+                    .setOxmList(oxmListSlash32).build();
+            OFAction setDmac = null, outg = null;
+            OFOxmEthDst dmac = factory.oxms()
+                    .ethDst(MacAddress.of(hostNextHopIps.get(i).dstMac));
+            setDmac = factory.actions().buildSetField()
+                    .setField(dmac).build();
+            outg = factory.actions().buildGroup()
+                    .setGroup(OFGroup.of(0x20000000 | hostNextHopIps.get(i).nextHopPort)) // l3group
+                                                                                          // id
+                    .build();
+            List<OFAction> writeActions = new ArrayList<OFAction>();
+            writeActions.add(setDmac);
+            writeActions.add(outg);
+            OFInstruction writeInstr = factory.instructions().buildWriteActions()
+                    .setActions(writeActions).build();
+            OFInstruction gotoInstr = factory.instructions().buildGotoTable()
+                    .setTableId(TableId.of(TABLE_ACL)).build();
+            List<OFInstruction> instructions = new ArrayList<OFInstruction>();
+            instructions.add(writeInstr);
+            instructions.add(gotoInstr);
+            OFMessage myIpEntry = factory.buildFlowAdd()
+                    .setTableId(TableId.of(TABLE_IPV4_UNICAST))
+                    .setMatch(match)
+                    .setInstructions(instructions)
+                    .setPriority(MAX_PRIORITY) // highest priority for exact
+                                               // match
+                    .setBufferId(OFBufferId.NO_BUFFER)
+                    .setIdleTimeout(0)
+                    .setHardTimeout(0)
+                    .setXid(getNextTransactionId())
+                    .build();
+            msglist.add(myIpEntry);
+        }
+        write(msglist);
+        log.debug("Adding {} next-hop-host-rules in sw {}", msglist.size(), getStringId());
+    }
+
+    private static class MplsEntry {
+        int labelid;
+        int portnum;
+
+        public MplsEntry(int labelid, int portnum) {
+            this.labelid = labelid;
+            this.portnum = portnum;
+        }
+    }
+
+    private List<MplsEntry> getMplsEntries() {
+        List<MplsEntry> myLabels = new ArrayList<MplsEntry>();
+        if (getId() == 0x1) {
+            myLabels.add(new MplsEntry(101, OFPort.CONTROLLER.getPortNumber()));
+            myLabels.add(new MplsEntry(103, 6));
+        }
+        if (getId() == 0x2) {
+            myLabels.add(new MplsEntry(103, 2));
+            myLabels.add(new MplsEntry(102, OFPort.CONTROLLER.getPortNumber()));
+            myLabels.add(new MplsEntry(101, 1));
+        }
+        if (getId() == 0x3) {
+            myLabels.add(new MplsEntry(103, OFPort.CONTROLLER.getPortNumber()));
+            myLabels.add(new MplsEntry(101, 2));
+        }
+        return myLabels;
+    }
+
+    private void populateMplsTable() throws IOException {
+        List<OFMessage> msglist = new ArrayList<OFMessage>();
+        List<MplsEntry> lfibEntries = getMplsEntries();
+        for (int i = 0; i < lfibEntries.size(); i++) {
+            OFOxmEthType ethTypeMpls = factory.oxms()
+                    .ethType(EthType.MPLS_UNICAST);
+            OFOxmMplsLabel labelid = factory.oxms()
+                    .mplsLabel(U32.of(lfibEntries.get(i).labelid));
+            OFOxmList oxmList = OFOxmList.of(ethTypeMpls, labelid);
+            OFMatchV3 matchlabel = factory.buildMatchV3()
+                    .setOxmList(oxmList).build();
+            OFAction poplabel = factory.actions().popMpls(EthType.IPv4);
+            OFAction sendTo = null;
+            if (lfibEntries.get(i).portnum == OFPort.CONTROLLER.getPortNumber()) {
+                sendTo = factory.actions().output(OFPort.CONTROLLER,
+                        OFPCML_NO_BUFFER);
+            } else {
+                sendTo = factory.actions().group(OFGroup.of(
+                        0xa0000000 | lfibEntries.get(i).portnum));
+            }
+            List<OFAction> writeActions = new ArrayList<OFAction>();
+            writeActions.add(poplabel);
+            writeActions.add(sendTo);
+            OFInstruction writeInstr = factory.instructions().buildWriteActions()
+                    .setActions(writeActions).build();
+            OFInstruction gotoInstr = factory.instructions().buildGotoTable()
+                    .setTableId(TableId.of(TABLE_ACL)).build();
+            List<OFInstruction> instructions = new ArrayList<OFInstruction>();
+            instructions.add(writeInstr);
+            instructions.add(gotoInstr);
+            OFMessage myMplsEntry = factory.buildFlowAdd()
+                    .setTableId(TableId.of(TABLE_MPLS))
+                    .setMatch(matchlabel)
+                    .setInstructions(instructions)
+                    .setPriority(MAX_PRIORITY) // exact match and exclusive
+                    .setBufferId(OFBufferId.NO_BUFFER)
+                    .setIdleTimeout(0)
+                    .setHardTimeout(0)
+                    .setXid(getNextTransactionId())
+                    .build();
+            msglist.add(myMplsEntry);
+        }
+        write(msglist);
+        log.debug("Adding {} mpls-forwarding-rules in sw {}", msglist.size(),
+                getStringId());
+
+        // match for everything else to send to ACL table. Essentially
+        // the table miss flow entry
+        populateTableMissEntry(TABLE_MPLS, false, true,
+                true, TABLE_ACL);
+
+    }
+
+    /**
+     * By default if none of the booleans in the call are set, then the
+     * table-miss entry is added with no instructions, which means that pipeline
+     * execution will stop, and the action set associated with the packet will
+     * be executed.
+     *
+     * @param tableToAdd
+     * @param toControllerNow as an APPLY_ACTION instruction
+     * @param toControllerWrite as a WRITE_ACITION instruction
+     * @param toTable as a GOTO_TABLE instruction
+     * @param tableToSend
+     * @throws IOException
+     */
+    @SuppressWarnings("unchecked")
+    private void populateTableMissEntry(int tableToAdd, boolean toControllerNow,
+            boolean toControllerWrite,
+            boolean toTable, int tableToSend) throws IOException {
+        OFOxmList oxmList = OFOxmList.EMPTY;
+        OFMatchV3 match = factory.buildMatchV3()
+                .setOxmList(oxmList)
+                .build();
+        OFAction outc = factory.actions()
+                .buildOutput()
+                .setPort(OFPort.CONTROLLER)
+                .setMaxLen(OFPCML_NO_BUFFER)
+                .build();
+        List<OFInstruction> instructions = new ArrayList<OFInstruction>();
+        if (toControllerNow) {
+            // table-miss instruction to send to controller immediately
+            OFInstruction instr = factory.instructions()
+                    .buildApplyActions()
+                    .setActions(Collections.singletonList(outc))
+                    .build();
+            instructions.add(instr);
+        }
+
+        if (toControllerWrite) {
+            // table-miss instruction to write-action to send to controller
+            // this will be executed whenever the action-set gets executed
+            OFInstruction instr = factory.instructions()
+                    .buildWriteActions()
+                    .setActions(Collections.singletonList(outc))
+                    .build();
+            instructions.add(instr);
+        }
+
+        if (toTable) {
+            // table-miss instruction to goto-table x
+            OFInstruction instr = factory.instructions()
+                    .gotoTable(TableId.of(tableToSend));
+            instructions.add(instr);
+        }
+
+        if (!toControllerNow && !toControllerWrite && !toTable) {
+            // table-miss has no instruction - at which point action-set will be
+            // executed - if there is an action to output/group in the action
+            // set
+            // the packet will be sent there, otherwise it will be dropped.
+            instructions = (List<OFInstruction>) Collections.EMPTY_LIST;
+        }
+
+        OFMessage tableMissEntry = factory.buildFlowAdd()
+                .setTableId(TableId.of(tableToAdd))
+                .setMatch(match) // match everything
+                .setInstructions(instructions)
+                .setPriority(MIN_PRIORITY)
+                .setBufferId(OFBufferId.NO_BUFFER)
+                .setIdleTimeout(0)
+                .setHardTimeout(0)
+                .setXid(getNextTransactionId())
+                .build();
+        write(tableMissEntry, null);
+    }
+
+    private void sendBarrier(boolean finalBarrier) throws IOException {
+        int xid = getNextTransactionId();
+        if (finalBarrier) {
+            barrierXidToWaitFor = xid;
+        }
+        OFBarrierRequest br = factory
+                .buildBarrierRequest()
+                .setXid(xid)
+                .build();
+        write(br, null);
+    }
+
+}
diff --git a/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplOVS10.java b/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplOVS10.java
new file mode 100644
index 0000000..bfe613e
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplOVS10.java
@@ -0,0 +1,29 @@
+package net.onrc.onos.core.drivermanager;
+
+import net.floodlightcontroller.core.internal.OFSwitchImplBase;
+
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+
+/**
+ * OFDescriptionStatistics Vendor (Manufacturer Desc.): Nicira, Inc. Make
+ * (Hardware Desc.) : Open vSwitch Model (Datapath Desc.) : None Software :
+ * 1.11.90 (or whatever version + build) Serial : None
+ */
+public class OFSwitchImplOVS10 extends OFSwitchImplBase {
+
+    public OFSwitchImplOVS10(OFDescStatsReply desc) {
+        super();
+        setSwitchDescription(desc);
+        setAttribute(SWITCH_SUPPORTS_NX_ROLE, true);
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        return "OFSwitchImplOVS10 [" + ((channel != null)
+                ? channel.getRemoteAddress() : "?")
+                + " DPID[" + ((stringId != null) ? stringId : "?") + "]]";
+    }
+}
diff --git a/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplOVS13.java b/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplOVS13.java
new file mode 100644
index 0000000..efe43a0
--- /dev/null
+++ b/src/main/java/net/onrc/onos/core/drivermanager/OFSwitchImplOVS13.java
@@ -0,0 +1,138 @@
+package net.onrc.onos.core.drivermanager;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeAlreadyStarted;
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeCompleted;
+import net.floodlightcontroller.core.SwitchDriverSubHandshakeNotStarted;
+import net.floodlightcontroller.core.internal.OFSwitchImplBase;
+
+import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFErrorMsg;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+
+/**
+ * OFDescriptionStatistics Vendor (Manufacturer Desc.): Nicira, Inc. Make
+ * (Hardware Desc.) : Open vSwitch Model (Datapath Desc.) : None Software :
+ * 2.1.0 (or whatever version + build) Serial : None
+ */
+public class OFSwitchImplOVS13 extends OFSwitchImplBase {
+    private AtomicBoolean driverHandshakeComplete;
+    private OFFactory factory;
+    private long barrierXidToWaitFor = -1;
+
+    public OFSwitchImplOVS13(OFDescStatsReply desc) {
+        super();
+        driverHandshakeComplete = new AtomicBoolean(false);
+        setSwitchDescription(desc);
+    }
+
+    @Override
+    public String toString() {
+        return "OFSwitchImplOVS13 [" + ((channel != null)
+                ? channel.getRemoteAddress() : "?")
+                + " DPID[" + ((stringId != null) ? stringId : "?") + "]]";
+    }
+
+    @Override
+    public void startDriverHandshake() throws IOException {
+        log.debug("Starting driver handshake for sw {}", getStringId());
+        if (startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeAlreadyStarted();
+        }
+        startDriverHandshakeCalled = true;
+        factory = floodlightProvider.getOFMessageFactory_13();
+        configureSwitch();
+    }
+
+    @Override
+    public boolean isDriverHandshakeComplete() {
+        if (!startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeNotStarted();
+        }
+        return driverHandshakeComplete.get();
+    }
+
+    @Override
+    public void processDriverHandshakeMessage(OFMessage m) {
+        if (!startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeNotStarted();
+        }
+        if (driverHandshakeComplete.get()) {
+            throw new SwitchDriverSubHandshakeCompleted(m);
+        }
+
+        switch (m.getType()) {
+        case BARRIER_REPLY:
+            if (m.getXid() == barrierXidToWaitFor) {
+                driverHandshakeComplete.set(true);
+            }
+            break;
+
+        case ERROR:
+            log.error("Switch {} Error {}", getStringId(), (OFErrorMsg) m);
+            break;
+
+        case FEATURES_REPLY:
+            break;
+        case FLOW_REMOVED:
+            break;
+        case GET_ASYNC_REPLY:
+            // OFAsyncGetReply asrep = (OFAsyncGetReply)m;
+            // decodeAsyncGetReply(asrep);
+            break;
+
+        case PACKET_IN:
+            break;
+        case PORT_STATUS:
+            break;
+        case QUEUE_GET_CONFIG_REPLY:
+            break;
+        case ROLE_REPLY:
+            break;
+
+        case STATS_REPLY:
+            // processStatsReply((OFStatsReply) m);
+            break;
+
+        default:
+            log.debug("Received message {} during switch-driver subhandshake "
+                    + "from switch {} ... Ignoring message", m, getStringId());
+
+        }
+    }
+
+
+    private void configureSwitch() throws IOException {
+        // setAsyncConfig();
+        // getTableFeatures();
+        /*sendGroupFeaturesRequest();
+        setL2Groups();
+        sendBarrier(false);
+        setL3Groups();
+        setL25Groups();
+        sendGroupDescRequest();*/
+        // populateTableVlan();
+        // populateTableTMac();
+        // populateIpTable();
+        // populateMplsTable();
+        // populateTableMissEntry(TABLE_ACL, false, false, false, -1);
+        sendBarrier(true);
+    }
+
+
+    private void sendBarrier(boolean finalBarrier) throws IOException {
+        int xid = getNextTransactionId();
+        if (finalBarrier) {
+            barrierXidToWaitFor = xid;
+        }
+        OFBarrierRequest br = factory
+                .buildBarrierRequest()
+                .setXid(xid)
+                .build();
+        write(br, null);
+    }
+}
diff --git a/src/main/java/net/onrc/onos/core/flowprogrammer/FlowProgrammer.java b/src/main/java/net/onrc/onos/core/flowprogrammer/FlowProgrammer.java
index 046d130..4a62947 100644
--- a/src/main/java/net/onrc/onos/core/flowprogrammer/FlowProgrammer.java
+++ b/src/main/java/net/onrc/onos/core/flowprogrammer/FlowProgrammer.java
@@ -7,6 +7,7 @@
 
 import net.floodlightcontroller.core.IFloodlightProviderService;
 import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.IOFSwitch.PortChangeType;
 import net.floodlightcontroller.core.IOFSwitchListener;
 import net.floodlightcontroller.core.module.FloodlightModuleContext;
 import net.floodlightcontroller.core.module.FloodlightModuleException;
@@ -16,19 +17,20 @@
 import net.onrc.onos.core.flowprogrammer.web.FlowProgrammerWebRoutable;
 import net.onrc.onos.core.registry.IControllerRegistryService;
 
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * FlowProgrammer is a module responsible to maintain flows installed to switches.
- * FlowProgrammer consists of FlowPusher and FlowSynchronizer.
- * FlowPusher manages the rate of installation, and FlowSynchronizer synchronizes
- * flows between GraphDB and switches.
- * FlowProgrammer also watch the event of addition/deletion of switches to
- * start/stop synchronization. When a switch is added to network, FlowProgrammer
- * immediately kicks synchronization to keep switch's flow table latest state.
- * Adversely, when a switch is removed from network, FlowProgrammer immediately
- * stops synchronization.
+ * FlowProgrammer is a module responsible to maintain flows installed to
+ * switches. FlowProgrammer consists of FlowPusher and FlowSynchronizer.
+ * FlowPusher manages the rate of installation, and FlowSynchronizer
+ * synchronizes flows between GraphDB and switches. FlowProgrammer also watch
+ * the event of addition/deletion of switches to start/stop synchronization.
+ * When a switch is added to network, FlowProgrammer immediately kicks
+ * synchronization to keep switch's flow table latest state. Adversely, when a
+ * switch is removed from network, FlowProgrammer immediately stops
+ * synchronization.
  */
 public class FlowProgrammer implements IFloodlightModule,
         IOFSwitchListener {
@@ -57,7 +59,7 @@
         floodlightProvider = context.getServiceImpl(IFloodlightProviderService.class);
         registryService = context.getServiceImpl(IControllerRegistryService.class);
         restApi = context.getServiceImpl(IRestApiService.class);
-        pusher.init(null, context, floodlightProvider.getOFMessageFactory(), null);
+        pusher.init(context);
         if (ENABLE_FLOW_SYNC) {
             synchronizer.init(pusher);
         }
@@ -83,8 +85,7 @@
 
     @Override
     public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() {
-        Map<Class<? extends IFloodlightService>,
-                IFloodlightService> m =
+        Map<Class<? extends IFloodlightService>, IFloodlightService> m =
                 new HashMap<Class<? extends IFloodlightService>,
                         IFloodlightService>();
         m.put(IFlowPusherService.class, pusher);
@@ -103,25 +104,56 @@
         return l;
     }
 
+    // ***********************
+    // IOFSwitchListener
+    // ***********************
+
     @Override
     public String getName() {
-        // TODO Auto-generated method stub
         return "FlowProgrammer";
     }
 
     @Override
-    public void addedSwitch(IOFSwitch sw) {
-        log.debug("Switch added: {}", sw.getId());
+    public void switchActivatedMaster(long swId) {
+        IOFSwitch sw = floodlightProvider.getSwitch(swId);
+        if (sw == null) {
+            log.warn("Added switch not available {} ", swId);
+            return;
+        }
+        log.debug("Switch added: {}", swId);
 
         if (ENABLE_FLOW_SYNC) {
-            if (registryService.hasControl(sw.getId())) {
+            if (registryService.hasControl(swId)) {
                 synchronizer.synchronize(sw);
             }
         }
     }
 
     @Override
-    public void removedSwitch(IOFSwitch sw) {
+    public void switchActivatedEqual(long swId) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void switchMasterToEqual(long swId) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void switchEqualToMaster(long swId) {
+        // for now treat as switchActivatedMaster
+        switchActivatedMaster(swId);
+    }
+
+    @Override
+    public void switchDisconnected(long swId) {
+        IOFSwitch sw = floodlightProvider.getSwitch(swId);
+        if (sw == null) {
+            log.warn("Removed switch not available {} ", swId);
+            return;
+        }
         log.debug("Switch removed: {}", sw.getId());
 
         if (ENABLE_FLOW_SYNC) {
@@ -132,7 +164,7 @@
     }
 
     @Override
-    public void switchPortChanged(Long switchId) {
+    public void switchPortChanged(long swId, OFPortDesc port, PortChangeType pct) {
         // TODO Auto-generated method stub
     }
 
diff --git a/src/main/java/net/onrc/onos/core/flowprogrammer/FlowPusher.java b/src/main/java/net/onrc/onos/core/flowprogrammer/FlowPusher.java
index 37e1b44..17bbc1a 100644
--- a/src/main/java/net/onrc/onos/core/flowprogrammer/FlowPusher.java
+++ b/src/main/java/net/onrc/onos/core/flowprogrammer/FlowPusher.java
@@ -4,7 +4,6 @@
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
@@ -25,48 +24,17 @@
 import net.floodlightcontroller.core.internal.OFMessageFuture;
 import net.floodlightcontroller.core.module.FloodlightModuleContext;
 import net.floodlightcontroller.threadpool.IThreadPoolService;
-import net.floodlightcontroller.util.MACAddress;
 import net.floodlightcontroller.util.OFMessageDamper;
-import net.onrc.onos.core.util.FlowEntry;
-import net.onrc.onos.core.util.FlowEntryAction;
-import net.onrc.onos.core.util.FlowEntryAction.ActionEnqueue;
-import net.onrc.onos.core.util.FlowEntryAction.ActionOutput;
-import net.onrc.onos.core.util.FlowEntryAction.ActionSetEthernetAddr;
-import net.onrc.onos.core.util.FlowEntryAction.ActionSetIPv4Addr;
-import net.onrc.onos.core.util.FlowEntryAction.ActionSetIpToS;
-import net.onrc.onos.core.util.FlowEntryAction.ActionSetTcpUdpPort;
-import net.onrc.onos.core.util.FlowEntryAction.ActionSetVlanId;
-import net.onrc.onos.core.util.FlowEntryAction.ActionSetVlanPriority;
-import net.onrc.onos.core.util.FlowEntryAction.ActionStripVlan;
-import net.onrc.onos.core.util.FlowEntryActions;
-import net.onrc.onos.core.util.FlowEntryMatch;
-import net.onrc.onos.core.util.FlowEntryUserState;
-import net.onrc.onos.core.util.IPv4Net;
+import net.onrc.onos.core.intent.FlowEntry;
 import net.onrc.onos.core.util.Pair;
-import net.onrc.onos.core.util.PortNumber;
 
-import org.openflow.protocol.OFBarrierReply;
-import org.openflow.protocol.OFBarrierRequest;
-import org.openflow.protocol.OFFlowMod;
-import org.openflow.protocol.OFMatch;
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFPacketOut;
-import org.openflow.protocol.OFPort;
-import org.openflow.protocol.OFType;
-import org.openflow.protocol.action.OFAction;
-import org.openflow.protocol.action.OFActionDataLayerDestination;
-import org.openflow.protocol.action.OFActionDataLayerSource;
-import org.openflow.protocol.action.OFActionEnqueue;
-import org.openflow.protocol.action.OFActionNetworkLayerDestination;
-import org.openflow.protocol.action.OFActionNetworkLayerSource;
-import org.openflow.protocol.action.OFActionNetworkTypeOfService;
-import org.openflow.protocol.action.OFActionOutput;
-import org.openflow.protocol.action.OFActionStripVirtualLan;
-import org.openflow.protocol.action.OFActionTransportLayerDestination;
-import org.openflow.protocol.action.OFActionTransportLayerSource;
-import org.openflow.protocol.action.OFActionVirtualLanIdentifier;
-import org.openflow.protocol.action.OFActionVirtualLanPriorityCodePoint;
-import org.openflow.protocol.factory.BasicFactory;
+import org.projectfloodlight.openflow.protocol.OFBarrierReply;
+import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFFlowMod;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFVersion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -75,14 +43,13 @@
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 
 /**
- * FlowPusher is a implementation of FlowPusherService.
- * FlowPusher assigns one message queue instance for each one switch.
- * Number of message processing threads is configurable by constructor, and
- * one thread can handle multiple message queues. Each queue will be assigned to
- * a thread according to hash function defined by getHash().
- * Each processing thread reads messages from queues and sends it to switches
- * in round-robin. Processing thread also calculates rate of sending to suppress
- * excessive message sending.
+ * FlowPusher is a implementation of FlowPusherService. FlowPusher assigns one
+ * message queue instance for each one switch. Number of message processing
+ * threads is configurable by constructor, and one thread can handle multiple
+ * message queues. Each queue will be assigned to a thread according to hash
+ * function defined by getHash(). Each processing thread reads messages from
+ * queues and sends it to switches in round-robin. Processing thread also
+ * calculates rate of sending to suppress excessive message sending.
  */
 public final class FlowPusher implements IFlowPusherService, IOFMessageListener {
     private static final Logger log = LoggerFactory.getLogger(FlowPusher.class);
@@ -91,8 +58,9 @@
     // TODO: Values copied from elsewhere (class LearningSwitch).
     // The local copy should go away!
     //
-    protected static final int OFMESSAGE_DAMPER_CAPACITY = 10000; // TODO: find sweet spot
-    protected static final int OFMESSAGE_DAMPER_TIMEOUT = 250;    // ms
+    protected static final int OFMESSAGE_DAMPER_CAPACITY = 10000; // TODO: find
+                                                                  // sweet spot
+    protected static final int OFMESSAGE_DAMPER_TIMEOUT = 250; // ms
 
     // Number of messages sent to switch at once
     protected static final int MAX_MESSAGE_SEND = 100;
@@ -110,15 +78,15 @@
     }
 
     /**
-     * SwitchQueue represents message queue attached to a switch.
-     * This consists of queue itself and variables used for limiting sending rate.
+     * SwitchQueue represents message queue attached to a switch. This consists
+     * of queue itself and variables used for limiting sending rate.
      */
     private static class SwitchQueue {
         List<Queue<SwitchQueueEntry>> rawQueues;
         QueueState state;
 
         // Max rate of sending message (bytes/ms). 0 implies no limitation.
-        long maxRate = 0;    // 0 indicates no limitation
+        long maxRate = 0; // 0 indicates no limitation
         long lastSentTime = 0;
         long lastSentSize = 0;
 
@@ -137,7 +105,7 @@
 
         /**
          * Check if sending rate is within the rate.
-         *
+         * <p>
          * @param current Current time
          * @return true if within the rate
          */
@@ -158,9 +126,9 @@
 
         /**
          * Log time and size of last sent data.
-         *
+         * <p>
          * @param current Time to be sent.
-         * @param size    Size of sent data (in bytes).
+         * @param size Size of sent data (in bytes).
          */
         void logSentData(long current, long size) {
             lastSentTime = current;
@@ -178,52 +146,52 @@
 
         /**
          * Poll single appropriate entry object according to QueueState.
-         *
+         * <p>
          * @return Entry object.
          */
         SwitchQueueEntry poll() {
             switch (state) {
-                case READY: {
-                    for (int i = 0; i < rawQueues.size(); ++i) {
-                        SwitchQueueEntry entry = rawQueues.get(i).poll();
-                        if (entry != null) {
-                            return entry;
-                        }
+            case READY: {
+                for (int i = 0; i < rawQueues.size(); ++i) {
+                    SwitchQueueEntry entry = rawQueues.get(i).poll();
+                    if (entry != null) {
+                        return entry;
                     }
+                }
 
-                    return null;
-                }
-                case SUSPENDED: {
-                    // Only polling from high priority queue
-                    SwitchQueueEntry entry = getQueue(MsgPriority.HIGH).poll();
-                    return entry;
-                }
-                default:
-                    log.error("Unexpected QueueState: {}", state);
-                    return null;
+                return null;
+            }
+            case SUSPENDED: {
+                // Only polling from high priority queue
+                SwitchQueueEntry entry = getQueue(MsgPriority.HIGH).poll();
+                return entry;
+            }
+            default:
+                log.error("Unexpected QueueState: {}", state);
+                return null;
             }
         }
 
         /**
          * Check if this object has any messages in the queues to be sent.
-         *
+         * <p>
          * @return True if there are some messages to be sent.
          */
         boolean hasMessageToSend() {
             switch (state) {
-                case READY:
-                    for (Queue<SwitchQueueEntry> queue : rawQueues) {
-                        if (!queue.isEmpty()) {
-                            return true;
-                        }
+            case READY:
+                for (Queue<SwitchQueueEntry> queue : rawQueues) {
+                    if (!queue.isEmpty()) {
+                        return true;
                     }
-                    break;
-                case SUSPENDED:
-                    // Only checking high priority queue
-                    return (!getQueue(MsgPriority.HIGH).isEmpty());
-                default:
-                    log.error("Unexpected QueueState: {}", state);
-                    return false;
+                }
+                break;
+            case SUSPENDED:
+                // Only checking high priority queue
+                return (!getQueue(MsgPriority.HIGH).isEmpty());
+            default:
+                log.error("Unexpected QueueState: {}", state);
+                return false;
             }
 
             return false;
@@ -239,7 +207,7 @@
      */
     private static final class BarrierInfo {
         final long dpid;
-        final int xid;
+        final long xid;
 
         static BarrierInfo create(IOFSwitch sw, OFBarrierRequest req) {
             return new BarrierInfo(sw.getId(), req.getXid());
@@ -249,7 +217,7 @@
             return new BarrierInfo(sw.getId(), rpy.getXid());
         }
 
-        private BarrierInfo(long dpid, int xid) {
+        private BarrierInfo(long dpid, long xid) {
             this.dpid = dpid;
             this.xid = xid;
         }
@@ -260,7 +228,7 @@
             final int prime = 31;
             int result = 1;
             result = prime * result + (int) (dpid ^ (dpid >>> 32));
-            result = prime * result + xid;
+            result = prime * result + (int) (xid ^ (xid >>> 32));
             return result;
         }
 
@@ -280,32 +248,30 @@
             return (this.dpid == other.dpid) && (this.xid == other.xid);
         }
 
-
     }
 
+    private FloodlightModuleContext context = null;
     private OFMessageDamper messageDamper = null;
     private IThreadPoolService threadPool = null;
-
-    private FloodlightContext context = null;
-    private BasicFactory factory = null;
+    private IFloodlightProviderService floodlightProvider = null;
+    protected Map<OFVersion, OFFactory> ofFactoryMap = null;
 
     // Map of threads versus dpid
     private Map<Long, FlowPusherThread> threadMap = null;
     // Map from (DPID and transaction ID) to Future objects.
-    private Map<BarrierInfo, OFBarrierReplyFuture> barrierFutures
-            = new ConcurrentHashMap<BarrierInfo, OFBarrierReplyFuture>();
+    private Map<BarrierInfo, OFBarrierReplyFuture> barrierFutures =
+            new ConcurrentHashMap<BarrierInfo, OFBarrierReplyFuture>();
 
     private int numberThread;
 
     /**
      * Main thread that reads messages from queues and sends them to switches.
      */
-    private class FlowPusherThread extends Thread {
-        // Weak ConncurrentHashMap
-        private Map<IOFSwitch, SwitchQueue> assignedQueues
-                = CacheBuilder.newBuilder()
-                    .weakKeys()
-                    .<IOFSwitch, SwitchQueue>build().asMap();
+    private static class FlowPusherThread extends Thread {
+        // Weak ConcurrentHashMap
+        private Map<IOFSwitch, SwitchQueue> assignedQueues = CacheBuilder.newBuilder()
+                .weakKeys()
+                .<IOFSwitch, SwitchQueue>build().asMap();
 
         final Lock queuingLock = new ReentrantLock();
         final Condition messagePushed = queuingLock.newCondition();
@@ -329,9 +295,8 @@
                     }
                 }
 
-                for (Iterator<Entry<IOFSwitch, SwitchQueue>> it = assignedQueues.entrySet().iterator();
-                        it.hasNext();
-                        ) {
+                for (Iterator<Entry<IOFSwitch, SwitchQueue>> it = assignedQueues
+                        .entrySet().iterator(); it.hasNext();) {
                     Entry<IOFSwitch, SwitchQueue> entry = it.next();
                     IOFSwitch sw = entry.getKey();
                     SwitchQueue queue = entry.getValue();
@@ -352,13 +317,13 @@
         }
 
         /**
-         * Read messages from queue and send them to the switch.
-         * If number of messages excess the limit, stop sending messages.
-         *
-         * @param sw      Switch to which messages will be sent.
-         * @param queue   Queue of messages.
-         * @param maxMsg Limitation of number of messages to be sent. If set to 0,
-         *                all messages in queue will be sent.
+         * Read messages from queue and send them to the switch. If number of
+         * messages excess the limit, stop sending messages.
+         * <p>
+         * @param sw Switch to which messages will be sent.
+         * @param queue Queue of messages.
+         * @param maxMsg Limitation of number of messages to be sent. If set to
+         *        0, all messages in queue will be sent.
          */
         private void processQueue(IOFSwitch sw, SwitchQueue queue, int maxMsg) {
             // check sending rate and determine it to be sent or not
@@ -381,11 +346,14 @@
 
                     OFMessage msg = queueEntry.getOFMessage();
                     try {
-                        messageDamper.write(sw, msg, context);
+                        // TODO BOC do we need to use the message damper?
+                        // messageDamper.write(sw, msg, context);
+                        sw.write(msg, null);
                         if (log.isTraceEnabled()) {
-                            log.trace("Pusher sends message: {}", msg);
+                            log.trace("Pusher sends message to switch {}: {}", sw.getStringId(), msg);
                         }
-                        size += msg.getLength();
+                        // TODO BOC how do we get the size?
+                        // size += msg.getLength();
                     } catch (IOException e) {
                         log.error("Exception in sending message (" + msg + "):", e);
                     }
@@ -425,7 +393,7 @@
 
     /**
      * Initialize object with threads of given number.
-     *
+     * <p>
      * @param numberThreadValue Number of threads to handle messages.
      */
     public FlowPusher(int numberThreadValue) {
@@ -438,44 +406,42 @@
 
     /**
      * Set parameters needed for sending messages.
-     *
-     * @param floodlightContext    FloodlightContext used for sending messages.
-     *                   If null, FlowPusher uses default context.
-     * @param modContext FloodlightModuleContext used for acquiring
-     *                   ThreadPoolService and registering MessageListener.
-     * @param basicFactory    Factory object to create OFMessage objects.
-     * @param damper     Message damper used for sending messages.
-     *                   If null, FlowPusher creates its own damper object.
+     * <p>
+     * @param floodlightContext FloodlightModuleContext used for acquiring
+     *        ThreadPoolService and registering MessageListener.
      */
-    public void init(FloodlightContext floodlightContext,
-                     FloodlightModuleContext modContext,
-                     BasicFactory basicFactory,
-                     OFMessageDamper damper) {
-        context = floodlightContext;
-        factory = basicFactory;
-        this.threadPool = modContext.getServiceImpl(IThreadPoolService.class);
-        IFloodlightProviderService flservice
-                = modContext.getServiceImpl(IFloodlightProviderService.class);
-        flservice.addOFMessageListener(OFType.BARRIER_REPLY, this);
+    public void init(FloodlightModuleContext floodlightContext) {
+        this.context = floodlightContext;
+        this.floodlightProvider = context
+                .getServiceImpl(IFloodlightProviderService.class);
+        this.threadPool = context.getServiceImpl(IThreadPoolService.class);
+        this.messageDamper = null;
 
-        if (damper != null) {
-            messageDamper = damper;
-        } else {
-            // use default values
-            messageDamper = new OFMessageDamper(OFMESSAGE_DAMPER_CAPACITY,
-                    EnumSet.of(OFType.FLOW_MOD),
-                    OFMESSAGE_DAMPER_TIMEOUT);
-        }
+        ofFactoryMap = new HashMap<>();
+        ofFactoryMap.put(OFVersion.OF_10, floodlightProvider.getOFMessageFactory_10());
+        ofFactoryMap.put(OFVersion.OF_13, floodlightProvider.getOFMessageFactory_13());
+        floodlightProvider.addOFMessageListener(OFType.BARRIER_REPLY, this);
+
+        // TODO BOC message damper may not be needed...
+        // if (damper != null) {
+        // messageDamper = damper;
+        // } else {
+        // use default values
+        /*messageDamper = new OFMessageDamper(OFMESSAGE_DAMPER_CAPACITY,
+                EnumSet.of(OFType.FLOW_MOD),
+                OFMESSAGE_DAMPER_TIMEOUT);*/
+        // }
     }
 
     /**
      * Begin processing queue.
      */
     public void start() {
-        if (factory == null) {
-            log.error("FlowPusher not yet initialized.");
-            return;
-        }
+        // TODO BOC
+        // if (factory == null) {
+        // log.error("FlowPusher not yet initialized.");
+        // return;
+        // }
 
         threadMap = new HashMap<Long, FlowPusherThread>();
         for (long i = 0; i < numberThread; ++i) {
@@ -491,7 +457,8 @@
         SwitchQueue queue = getQueue(sw);
 
         if (queue == null) {
-            // create queue in case suspend is called before first message addition
+            // create queue in case suspend is called before first message
+            // addition
             queue = createQueueImpl(sw);
         }
 
@@ -509,7 +476,7 @@
         SwitchQueue queue = getQueue(sw);
 
         if (queue == null) {
-            log.error("No queue is attached to DPID: {}", sw.getId());
+            log.error("No queue is attached to DPID: {}", sw.getStringId());
             return false;
         }
 
@@ -560,7 +527,7 @@
         }
 
         if (rate > 0) {
-            log.debug("rate for {} is set to {}", sw.getId(), rate);
+            log.debug("rate for {} is set to {}", sw.getStringId(), rate);
             synchronized (queue) {
                 queue.maxRate = rate;
             }
@@ -569,7 +536,7 @@
 
     @Override
     @SuppressFBWarnings(value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE",
-                        justification = "Future versions of createQueueImpl() might return null")
+            justification = "Future versions of createQueueImpl() might return null")
     public boolean createQueue(IOFSwitch sw) {
         SwitchQueue queue = createQueueImpl(sw);
 
@@ -619,13 +586,12 @@
 
     /**
      * Invalidate.
-     *
+     * <p>
      * @param sw switch
-     *
      * @see OFMessageDamper#invalidate(IOFSwitch)
      */
     public void invalidate(IOFSwitch sw) {
-        messageDamper.invalidate(sw);
+        // messageDamper.invalidate(sw); currently a null ptr - commenting out
     }
 
     @Override
@@ -668,226 +634,9 @@
     }
 
     /**
-     * Fetch the match conditions.
-     * NOTE: The Flow matching conditions common for all Flow Entries are
-     * used ONLY if a Flow Entry does NOT have the corresponding matching
-     * condition set.
-     *
-     * @param flowEntryMatch Flow entry to create a matcher for
-     * @return open flow matcher for the given values
-     */
-    private OFMatch computeMatch(FlowEntryMatch flowEntryMatch) {
-        OFMatch match = new OFMatch();
-        match.setWildcards(OFMatch.OFPFW_ALL);
-
-        // Match the Incoming Port
-        PortNumber matchInPort = flowEntryMatch.inPort();
-        if (matchInPort != null) {
-            match.setInputPort(matchInPort.shortValue());
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_IN_PORT);
-        }
-
-        // Match the Source MAC address
-        MACAddress matchSrcMac = flowEntryMatch.srcMac();
-        if (matchSrcMac != null) {
-            match.setDataLayerSource(matchSrcMac.toString());
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_DL_SRC);
-        }
-
-        // Match the Destination MAC address
-        MACAddress matchDstMac = flowEntryMatch.dstMac();
-        if (matchDstMac != null) {
-            match.setDataLayerDestination(matchDstMac.toString());
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_DL_DST);
-        }
-
-        // Match the Ethernet Frame Type
-        Short matchEthernetFrameType = flowEntryMatch.ethernetFrameType();
-        if (matchEthernetFrameType != null) {
-            match.setDataLayerType(matchEthernetFrameType);
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_DL_TYPE);
-        }
-
-        // Match the VLAN ID
-        Short matchVlanId = flowEntryMatch.vlanId();
-        if (matchVlanId != null) {
-            match.setDataLayerVirtualLan(matchVlanId);
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_DL_VLAN);
-        }
-
-        // Match the VLAN priority
-        Byte matchVlanPriority = flowEntryMatch.vlanPriority();
-        if (matchVlanPriority != null) {
-            match.setDataLayerVirtualLanPriorityCodePoint(matchVlanPriority);
-            match.setWildcards(match.getWildcards()
-                    & ~OFMatch.OFPFW_DL_VLAN_PCP);
-        }
-
-        // Match the Source IPv4 Network prefix
-        IPv4Net matchSrcIPv4Net = flowEntryMatch.srcIPv4Net();
-        if (matchSrcIPv4Net != null) {
-            match.setFromCIDR(matchSrcIPv4Net.toString(), OFMatch.STR_NW_SRC);
-        }
-
-        // Natch the Destination IPv4 Network prefix
-        IPv4Net matchDstIPv4Net = flowEntryMatch.dstIPv4Net();
-        if (matchDstIPv4Net != null) {
-            match.setFromCIDR(matchDstIPv4Net.toString(), OFMatch.STR_NW_DST);
-        }
-
-        // Match the IP protocol
-        Byte matchIpProto = flowEntryMatch.ipProto();
-        if (matchIpProto != null) {
-            match.setNetworkProtocol(matchIpProto);
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_NW_PROTO);
-        }
-
-        // Match the IP ToS (DSCP field, 6 bits)
-        Byte matchIpToS = flowEntryMatch.ipToS();
-        if (matchIpToS != null) {
-            match.setNetworkTypeOfService(matchIpToS);
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_NW_TOS);
-        }
-
-        // Match the Source TCP/UDP port
-        Short matchSrcTcpUdpPort = flowEntryMatch.srcTcpUdpPort();
-        if (matchSrcTcpUdpPort != null) {
-            match.setTransportSource(matchSrcTcpUdpPort);
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_TP_SRC);
-        }
-
-        // Match the Destination TCP/UDP port
-        Short matchDstTcpUdpPort = flowEntryMatch.dstTcpUdpPort();
-        if (matchDstTcpUdpPort != null) {
-            match.setTransportDestination(matchDstTcpUdpPort);
-            match.setWildcards(match.getWildcards() & ~OFMatch.OFPFW_TP_DST);
-        }
-
-        return match;
-    }
-
-
-    /**
-     * Wrapper object to hold a port number.  Used to pass around output ports.
-     */
-    private static class OutputPort {
-        private Short portNumber;
-    }
-
-    /**
-     * Process a flow action entry, putting the resulting flow
-     * actions into a list.  Will also set the actionOutputPort
-     * if one is encountered while processing an action.
-     *
-     * @param action Flow Entry Action to process
-     * @param openFlowActions actions to perform get added to this list
-     * @param actionOutputPort this will get set if an action output
-     *                         port is found
-     */
-    private void processAction(final FlowEntryAction action,
-                               final List<OFAction> openFlowActions,
-                               final OutputPort actionOutputPort) {
-        ActionOutput actionOutput = action.actionOutput();
-        ActionSetVlanId actionSetVlanId = action.actionSetVlanId();
-        ActionSetVlanPriority actionSetVlanPriority = action
-            .actionSetVlanPriority();
-        ActionStripVlan actionStripVlan = action.actionStripVlan();
-        ActionSetEthernetAddr actionSetEthernetSrcAddr = action
-            .actionSetEthernetSrcAddr();
-        ActionSetEthernetAddr actionSetEthernetDstAddr = action
-            .actionSetEthernetDstAddr();
-        ActionSetIPv4Addr actionSetIPv4SrcAddr = action
-            .actionSetIPv4SrcAddr();
-        ActionSetIPv4Addr actionSetIPv4DstAddr = action
-            .actionSetIPv4DstAddr();
-        ActionSetIpToS actionSetIpToS = action.actionSetIpToS();
-        ActionSetTcpUdpPort actionSetTcpUdpSrcPort = action
-            .actionSetTcpUdpSrcPort();
-        ActionSetTcpUdpPort actionSetTcpUdpDstPort = action
-            .actionSetTcpUdpDstPort();
-        ActionEnqueue actionEnqueue = action.actionEnqueue();
-
-        if (actionOutput != null) {
-            actionOutputPort.portNumber = actionOutput.port().shortValue();
-            // XXX: The max length is hard-coded for now
-            OFActionOutput ofa = new OFActionOutput(actionOutput.port()
-                                                    .shortValue(), (short) 0xffff);
-            openFlowActions.add(ofa);
-        }
-
-        if (actionSetVlanId != null) {
-            OFActionVirtualLanIdentifier ofa =
-                new OFActionVirtualLanIdentifier(actionSetVlanId.vlanId());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionSetVlanPriority != null) {
-            OFActionVirtualLanPriorityCodePoint ofa =
-                new OFActionVirtualLanPriorityCodePoint(actionSetVlanPriority.vlanPriority());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionStripVlan != null) {
-            if (actionStripVlan.stripVlan()) {
-                OFActionStripVirtualLan ofa = new OFActionStripVirtualLan();
-                openFlowActions.add(ofa);
-            }
-        }
-
-        if (actionSetEthernetSrcAddr != null) {
-            OFActionDataLayerSource ofa =
-                new OFActionDataLayerSource(actionSetEthernetSrcAddr.addr().toBytes());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionSetEthernetDstAddr != null) {
-            OFActionDataLayerDestination ofa =
-                new OFActionDataLayerDestination(actionSetEthernetDstAddr.addr().toBytes());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionSetIPv4SrcAddr != null) {
-            OFActionNetworkLayerSource ofa =
-                new OFActionNetworkLayerSource(actionSetIPv4SrcAddr.addr().value());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionSetIPv4DstAddr != null) {
-            OFActionNetworkLayerDestination ofa =
-                new OFActionNetworkLayerDestination(actionSetIPv4DstAddr.addr().value());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionSetIpToS != null) {
-            OFActionNetworkTypeOfService ofa =
-                new OFActionNetworkTypeOfService(actionSetIpToS.ipToS());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionSetTcpUdpSrcPort != null) {
-            OFActionTransportLayerSource ofa =
-                new OFActionTransportLayerSource(actionSetTcpUdpSrcPort.port());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionSetTcpUdpDstPort != null) {
-            OFActionTransportLayerDestination ofa =
-                new OFActionTransportLayerDestination(actionSetTcpUdpDstPort.port());
-            openFlowActions.add(ofa);
-        }
-
-        if (actionEnqueue != null) {
-            OFActionEnqueue ofa =
-                new OFActionEnqueue(actionEnqueue.port().shortValue(), actionEnqueue.queueId());
-            openFlowActions.add(ofa);
-        }
-    }
-
-
-    /**
      * Create a message from FlowEntry and add it to the queue of the switch.
-     *
-     * @param sw        Switch to which message is pushed.
+     * <p>
+     * @param sw Switch to which message is pushed.
      * @param flowEntry FlowEntry object used for creating message.
      * @return true if message is successfully added to a queue.
      */
@@ -895,90 +644,14 @@
         //
         // Create the OpenFlow Flow Modification Entry to push
         //
-        OFFlowMod fm = (OFFlowMod) factory.getMessage(OFType.FLOW_MOD);
-        long cookie = flowEntry.flowEntryId().value();
-
-        short flowModCommand = OFFlowMod.OFPFC_ADD;
-        if (flowEntry.flowEntryUserState() == FlowEntryUserState.FE_USER_ADD) {
-            flowModCommand = OFFlowMod.OFPFC_ADD;
-        } else if (flowEntry.flowEntryUserState() == FlowEntryUserState.FE_USER_MODIFY) {
-            flowModCommand = OFFlowMod.OFPFC_MODIFY_STRICT;
-        } else if (flowEntry.flowEntryUserState() == FlowEntryUserState.FE_USER_DELETE) {
-            flowModCommand = OFFlowMod.OFPFC_DELETE_STRICT;
-        } else {
-            // Unknown user state. Ignore the entry
-            log.debug(
-                    "Flow Entry ignored (FlowEntryId = {}): unknown user state {}",
-                    flowEntry.flowEntryId(),
-                    flowEntry.flowEntryUserState());
-            return false;
-        }
-
-        final FlowEntryMatch flowEntryMatch = flowEntry.flowEntryMatch();
-        final OFMatch match = computeMatch(flowEntryMatch);
-
-        final PortNumber matchInPort = flowEntryMatch.inPort();
-        final MACAddress matchSrcMac = flowEntryMatch.srcMac();
-        final MACAddress matchDstMac = flowEntryMatch.dstMac();
-
-        //
-        // Fetch the actions
-        //
-        final List<OFAction> openFlowActions = new ArrayList<OFAction>();
-        final FlowEntryActions flowEntryActions = flowEntry.flowEntryActions();
-        //
-        final OutputPort actionOutputPort = new OutputPort();
-        for (FlowEntryAction action : flowEntryActions.actions()) {
-            processAction(action, openFlowActions, actionOutputPort);
-        }
-        int actionsLen = 0;
-        for (OFAction ofa : openFlowActions) {
-            actionsLen += ofa.getLength();
-        }
-
-        fm.setIdleTimeout((short) flowEntry.idleTimeout())
-                .setHardTimeout((short) flowEntry.hardTimeout())
-                .setPriority((short) flowEntry.priority())
-                .setBufferId(OFPacketOut.BUFFER_ID_NONE).setCookie(cookie)
-                .setCommand(flowModCommand).setMatch(match)
-                .setActions(openFlowActions)
-                .setLengthU(OFFlowMod.MINIMUM_LENGTH + actionsLen);
-        fm.setOutPort(OFPort.OFPP_NONE.getValue());
-        if ((flowModCommand == OFFlowMod.OFPFC_DELETE)
-                || (flowModCommand == OFFlowMod.OFPFC_DELETE_STRICT)) {
-            if (actionOutputPort.portNumber != null) {
-                fm.setOutPort(actionOutputPort.portNumber);
-            }
-        }
-
-        //
-        // Set the OFPFF_SEND_FLOW_REM flag if the Flow Entry is not
-        // permanent.
-        //
-        if ((flowEntry.idleTimeout() != 0) ||
-                (flowEntry.hardTimeout() != 0)) {
-            fm.setFlags(OFFlowMod.OFPFF_SEND_FLOW_REM);
-        }
-
-        if (log.isTraceEnabled()) {
-            log.trace("Installing flow entry {} into switch DPID: {} " +
-                    "flowEntryId: {} srcMac: {} dstMac: {} inPort: {} outPort: {}"
-                    , flowEntry.flowEntryUserState()
-                    , sw.getStringId()
-                    , flowEntry.flowEntryId()
-                    , matchSrcMac
-                    , matchDstMac
-                    , matchInPort
-                    , actionOutputPort
-            );
-        }
-
+        OFFlowMod fm = flowEntry.buildFlowMod(ofFactoryMap.get(sw.getOFVersion()));
+        // log.trace("Pushing flow mod {}", fm);
         return addMessageImpl(sw, fm, priority);
     }
 
     /**
      * Add message to queue.
-     *
+     * <p>
      * @param sw
      * @param msg
      * @param priority
@@ -999,7 +672,7 @@
         synchronized (queue) {
             queue.add(entry, priority);
             if (log.isTraceEnabled()) {
-                log.trace("Message is pushed: {}", entry.getOFMessage());
+                log.trace("Message is pushed to switch {}: {}", sw.getStringId(), entry.getOFMessage());
             }
         }
 
@@ -1014,20 +687,18 @@
         if (future == null) {
             return null;
         }
-
         try {
             return future.get();
         } catch (InterruptedException e) {
             log.error("InterruptedException", e);
-            return null;
         } catch (ExecutionException e) {
             log.error("ExecutionException", e);
-            return null;
         }
+        return null;
     }
 
     @Override
-    public OFBarrierReplyFuture barrierAsync(IOFSwitch sw) {
+    public OFMessageFuture<OFBarrierReply> barrierAsync(IOFSwitch sw) {
         // TODO creation of message and future should be moved to OFSwitchImpl
 
         if (sw == null) {
@@ -1035,25 +706,28 @@
         }
 
         OFBarrierRequest msg = createBarrierRequest(sw);
-
-        OFBarrierReplyFuture future = new OFBarrierReplyFuture(threadPool, sw, msg.getXid());
+        OFBarrierReplyFuture future = new OFBarrierReplyFuture(threadPool, sw,
+                (int) msg.getXid());
         barrierFutures.put(BarrierInfo.create(sw, msg), future);
-
         addMessageImpl(sw, msg, MsgPriority.NORMAL);
-
         return future;
     }
 
     protected OFBarrierRequest createBarrierRequest(IOFSwitch sw) {
-        OFBarrierRequest msg = (OFBarrierRequest) factory.getMessage(OFType.BARRIER_REQUEST);
-        msg.setXid(sw.getNextTransactionId());
-
-        return msg;
+        OFFactory factory = ofFactoryMap.get(sw.getOFVersion());
+        if (factory == null) {
+            log.error("No OF Message Factory for switch {} with OFVersion {}", sw,
+                    sw.getOFVersion());
+            return null;
+        }
+        return factory.buildBarrierRequest()
+                .setXid(sw.getNextTransactionId())
+                .build();
     }
 
     /**
      * Get a queue attached to a switch.
-     *
+     * <p>
      * @param sw Switch object
      * @return Queue object
      */
@@ -1072,7 +746,7 @@
 
     /**
      * Get a hash value correspondent to a switch.
-     *
+     * <p>
      * @param sw Switch object
      * @return Hash value
      */
@@ -1084,7 +758,7 @@
 
     /**
      * Get a Thread object which processes the queue attached to a switch.
-     *
+     * <p>
      * @param sw Switch object
      * @return Thread object
      */
@@ -1112,18 +786,17 @@
     @Override
     public Command receive(IOFSwitch sw, OFMessage msg, FloodlightContext cntx) {
         if (log.isTraceEnabled()) {
-            log.trace("Received BARRIER_REPLY from: {}", sw.getId());
+            log.trace("Received BARRIER_REPLY from : {}", sw.getStringId());
         }
 
         if ((msg.getType() != OFType.BARRIER_REPLY) ||
-            !(msg instanceof OFBarrierReply)) {
+                !(msg instanceof OFBarrierReply)) {
             log.error("Unexpected reply message: {}", msg.getType());
             return Command.CONTINUE;
         }
 
         OFBarrierReply reply = (OFBarrierReply) msg;
         BarrierInfo info = BarrierInfo.create(sw, reply);
-
         // Deliver future if exists
         OFBarrierReplyFuture future = barrierFutures.get(info);
         if (future != null) {
@@ -1133,4 +806,5 @@
 
         return Command.CONTINUE;
     }
+
 }
diff --git a/src/main/java/net/onrc/onos/core/flowprogrammer/FlowSynchronizer.java b/src/main/java/net/onrc/onos/core/flowprogrammer/FlowSynchronizer.java
index af3cc70..7b2bfd0 100644
--- a/src/main/java/net/onrc/onos/core/flowprogrammer/FlowSynchronizer.java
+++ b/src/main/java/net/onrc/onos/core/flowprogrammer/FlowSynchronizer.java
@@ -1,6 +1,5 @@
 package net.onrc.onos.core.flowprogrammer;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -8,12 +7,10 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.FutureTask;
 
 import net.floodlightcontroller.core.IOFSwitch;
-import net.onrc.onos.core.flowprogrammer.IFlowPusherService.MsgPriority;
 import net.onrc.onos.core.util.FlowEntryId;
 
 import org.openflow.protocol.OFFlowMod;
@@ -99,10 +96,6 @@
                 Set<FlowEntryWrapper> graphEntries = getFlowEntriesFromGraph();
                 long step1 = System.nanoTime();
                 Set<FlowEntryWrapper> switchEntries = getFlowEntriesFromSwitch();
-                if (switchEntries == null) {
-                    log.debug("getFlowEntriesFromSwitch() failed");
-                    return null;
-                }
                 long step2 = System.nanoTime();
                 SyncResult result = compare(graphEntries, switchEntries);
                 long step3 = System.nanoTime();
@@ -184,12 +177,12 @@
             Set<FlowEntryWrapper> entries = new HashSet<FlowEntryWrapper>();
 
             // TODO: fix when FlowSynchronizer is refactored
-        /*
+            /*
         for(IFlowEntry entry : swObj.getFlowEntries()) {
         FlowEntryWrapper fe = new FlowEntryWrapper(entry);
         entries.add(fe);
         }
-        */
+             */
             return entries;
         }
 
@@ -218,8 +211,9 @@
             lengthU += req.getLengthU();
             req.setLengthU(lengthU);
 
-            List<OFStatistics> entries = null;
-            try {
+            //List<OFStatistics> entries = null;
+            // XXX S when we fix stats, we fix this
+            /*try {
                 Future<List<OFStatistics>> dfuture = sw.getStatistics(req);
                 entries = dfuture.get();
             } catch (IOException e) {
@@ -231,14 +225,16 @@
             } catch (ExecutionException e) {
                 log.error("Error getting statistics", e);
                 return null;
-            }
+            }*/
 
             Set<FlowEntryWrapper> results = new HashSet<FlowEntryWrapper>();
+            /*
             for (OFStatistics result : entries) {
                 OFFlowStatisticsReply entry = (OFFlowStatisticsReply) result;
                 FlowEntryWrapper fe = new FlowEntryWrapper(entry);
                 results.add(fe);
             }
+             */
             return results;
         }
 
@@ -248,7 +244,7 @@
      * FlowEntryWrapper represents abstract FlowEntry which is embodied
      * by FlowEntryId (from GraphDB) or OFFlowStatisticsReply (from switch).
      */
-    class FlowEntryWrapper {
+    static class FlowEntryWrapper {
         FlowEntryId flowEntryId;
         // TODO: fix when FlowSynchronizer is refactored
         // IFlowEntry iFlowEntry;
@@ -256,12 +252,12 @@
 
 
         // TODO: fix when FlowSynchronizer is refactored
-    /*
+        /*
     public FlowEntryWrapper(IFlowEntry entry) {
         flowEntryId = new FlowEntryId(entry.getFlowEntryId());
         iFlowEntry = entry;
     }
-    */
+         */
 
         public FlowEntryWrapper(OFFlowStatisticsReply entry) {
             flowEntryId = new FlowEntryId(entry.getCookie());
@@ -285,7 +281,7 @@
             double startDB = System.nanoTime();
             // Get the Flow Entry state from the Network Graph
             // TODO: fix when FlowSynchronizer is refactored
-        /*
+            /*
         if (iFlowEntry == null) {
             try {
         // TODO: fix when FlowSynchronizer is refactored
@@ -296,13 +292,13 @@
                 return;
             }
         }
-        */
+             */
             dbTime = System.nanoTime() - startDB;
 
             //
             // TODO: The old FlowDatabaseOperation class is gone, so the code
             //
-        /*
+            /*
         double startExtract = System.nanoTime();
         FlowEntry flowEntry =
         FlowDatabaseOperation.extractFlowEntry(iFlowEntry);
@@ -316,7 +312,7 @@
         double startPush = System.nanoTime();
         pusher.pushFlowEntry(sw, flowEntry, MsgPriority.HIGH);
         pushTime = System.nanoTime() - startPush;
-        */
+             */
         }
 
         /**
@@ -340,7 +336,8 @@
             fm.setPriority(statisticsReply.getPriority());
             fm.setOutPort(OFPort.OFPP_NONE);
 
-            pusher.add(sw, fm, MsgPriority.HIGH);
+            // XXX BOC commented out pending FlowSync refactor
+            //pusher.add(sw, fm, MsgPriority.HIGH);
         }
 
         /**
diff --git a/src/main/java/net/onrc/onos/core/flowprogrammer/IFlowPusherService.java b/src/main/java/net/onrc/onos/core/flowprogrammer/IFlowPusherService.java
index 37911a8..e94119a 100644
--- a/src/main/java/net/onrc/onos/core/flowprogrammer/IFlowPusherService.java
+++ b/src/main/java/net/onrc/onos/core/flowprogrammer/IFlowPusherService.java
@@ -5,11 +5,11 @@
 import net.floodlightcontroller.core.IOFSwitch;
 import net.floodlightcontroller.core.internal.OFMessageFuture;
 import net.floodlightcontroller.core.module.IFloodlightService;
-import net.onrc.onos.core.util.FlowEntry;
+import net.onrc.onos.core.intent.FlowEntry;
 import net.onrc.onos.core.util.Pair;
 
-import org.openflow.protocol.OFBarrierReply;
-import org.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFBarrierReply;
+import org.projectfloodlight.openflow.protocol.OFMessage;
 
 /**
  * FlowPusherService is a service to send message to switches in proper rate.
diff --git a/src/main/java/net/onrc/onos/core/flowprogrammer/OFBarrierReplyFuture.java b/src/main/java/net/onrc/onos/core/flowprogrammer/OFBarrierReplyFuture.java
index bbd10cb..3991019 100644
--- a/src/main/java/net/onrc/onos/core/flowprogrammer/OFBarrierReplyFuture.java
+++ b/src/main/java/net/onrc/onos/core/flowprogrammer/OFBarrierReplyFuture.java
@@ -6,22 +6,24 @@
 import net.floodlightcontroller.core.internal.OFMessageFuture;
 import net.floodlightcontroller.threadpool.IThreadPoolService;
 
-import org.openflow.protocol.OFBarrierReply;
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFBarrierReply;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFType;
+
+//XXX S note that other places are using old OFBarrierReply - so we broke that
 
 public class OFBarrierReplyFuture extends OFMessageFuture<OFBarrierReply> {
 
     protected volatile boolean finished;
 
     public OFBarrierReplyFuture(IThreadPoolService tp,
-                                IOFSwitch sw, int transactionId) {
+            IOFSwitch sw, int transactionId) {
         super(tp, sw, OFType.FEATURES_REPLY, transactionId);
         init();
     }
 
     public OFBarrierReplyFuture(IThreadPoolService tp,
-                                IOFSwitch sw, int transactionId, long timeout, TimeUnit unit) {
+            IOFSwitch sw, int transactionId, long timeout, TimeUnit unit) {
         super(tp, sw, OFType.FEATURES_REPLY, transactionId, timeout, unit);
         init();
     }
diff --git a/src/main/java/net/onrc/onos/core/flowprogrammer/web/SendBarrierResource.java b/src/main/java/net/onrc/onos/core/flowprogrammer/web/SendBarrierResource.java
index 236cc85..4fc6782 100644
--- a/src/main/java/net/onrc/onos/core/flowprogrammer/web/SendBarrierResource.java
+++ b/src/main/java/net/onrc/onos/core/flowprogrammer/web/SendBarrierResource.java
@@ -2,8 +2,8 @@
 
 import net.floodlightcontroller.core.IOFSwitch;
 
-import org.openflow.protocol.OFBarrierReply;
 import org.openflow.util.HexString;
+import org.projectfloodlight.openflow.protocol.OFBarrierReply;
 import org.restlet.resource.Get;
 
 /**
diff --git a/src/main/java/net/onrc/onos/core/hostmanager/HostManager.java b/src/main/java/net/onrc/onos/core/hostmanager/HostManager.java
index 6843fa9..9f6373d 100644
--- a/src/main/java/net/onrc/onos/core/hostmanager/HostManager.java
+++ b/src/main/java/net/onrc/onos/core/hostmanager/HostManager.java
@@ -30,15 +30,15 @@
 import net.onrc.onos.core.util.Dpid;
 import net.onrc.onos.core.util.PortNumber;
 
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFPacketIn;
-import org.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFPacketIn;
+import org.projectfloodlight.openflow.protocol.OFType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class HostManager implements IFloodlightModule,
-        IOFMessageListener,
-        IHostService {
+IOFMessageListener,
+IHostService {
 
     private static final Logger log = LoggerFactory.getLogger(HostManager.class);
     private static final long HOST_CLEANING_INITIAL_DELAY = 30;
@@ -106,8 +106,10 @@
 
             Ethernet eth = IFloodlightProviderService.bcStore.
                     get(cntx, IFloodlightProviderService.CONTEXT_PI_PAYLOAD);
+            short inport = (short) cntx.getStorage()
+                    .get(IFloodlightProviderService.CONTEXT_PI_INPORT);
 
-            return processPacketIn(sw, pi, eth);
+            return processPacketIn(sw, pi, eth, inport);
         }
 
         return Command.CONTINUE;
@@ -116,13 +118,14 @@
     // This "protected" modifier is for unit test.
     // The above "receive" method couldn't be tested
     // because of IFloodlightProviderService static final field.
-    protected Command processPacketIn(IOFSwitch sw, OFPacketIn pi, Ethernet eth) {
+    protected Command processPacketIn(IOFSwitch sw, OFPacketIn pi, Ethernet eth,
+            short inport) {
         if (log.isTraceEnabled()) {
             log.trace("Receive PACKET_IN swId {}, portId {}", sw.getId(), pi.getInPort());
         }
 
         final Dpid dpid = new Dpid(sw.getId());
-        final PortNumber portNum = new PortNumber(pi.getInPort());
+        final PortNumber portNum = new PortNumber(inport);
 
         Host srcHost =
                 getSourceHostFromPacket(eth, dpid.value(), portNum.value());
@@ -140,8 +143,8 @@
             if (topology.getOutgoingLink(dpid, portNum) != null ||
                     topology.getIncomingLink(dpid, portNum) != null) {
                 log.debug("Not adding host {} as " +
-                    "there is a link on the port: dpid {} port {}",
-                    srcHost.getMacAddress(), dpid, portNum);
+                        "there is a link on the port: dpid {} port {}",
+                        srcHost.getMacAddress(), dpid, portNum);
                 return Command.CONTINUE;
             }
         } finally {
diff --git a/src/main/java/net/onrc/onos/core/intent/Action.java b/src/main/java/net/onrc/onos/core/intent/Action.java
index 100dc33..8f8c8b2 100644
--- a/src/main/java/net/onrc/onos/core/intent/Action.java
+++ b/src/main/java/net/onrc/onos/core/intent/Action.java
@@ -2,6 +2,9 @@
 
 import net.onrc.onos.core.util.FlowEntryAction;
 
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+
 /**
  * An abstract class that represents an OpenFlow action.
  */
@@ -14,4 +17,12 @@
      * @return an equivalent FlowEntryAction object
      */
     public abstract FlowEntryAction getFlowEntryAction();
+
+    /**
+     * Builds and returns an OFAction given an OFFactory.
+     *
+     * @param factory the OFFactory to use for building
+     * @return the OFAction
+     */
+    public abstract OFAction getOFAction(OFFactory factory);
 }
diff --git a/src/main/java/net/onrc/onos/core/intent/FlowEntry.java b/src/main/java/net/onrc/onos/core/intent/FlowEntry.java
index c738cbc..0fd618e 100644
--- a/src/main/java/net/onrc/onos/core/intent/FlowEntry.java
+++ b/src/main/java/net/onrc/onos/core/intent/FlowEntry.java
@@ -1,22 +1,32 @@
 package net.onrc.onos.core.intent;
 
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
 import net.floodlightcontroller.util.MACAddress;
 import net.onrc.onos.core.intent.IntentOperation.Operator;
-import net.onrc.onos.core.util.Dpid;
-import net.onrc.onos.core.util.FlowEntryActions;
-import net.onrc.onos.core.util.FlowEntryId;
-import net.onrc.onos.core.util.FlowEntryUserState;
+
+import org.projectfloodlight.openflow.protocol.OFActionType;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFFlowMod;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.protocol.action.OFActionOutput;
+import org.projectfloodlight.openflow.protocol.match.Match.Builder;
+import org.projectfloodlight.openflow.types.OFBufferId;
+import org.projectfloodlight.openflow.types.OFPort;
+import org.projectfloodlight.openflow.types.U64;
 
 /**
- * A class to represent an OpenFlow FlowMod.
+ * A class to represent an OpenFlow FlowMod. <br>
  * It is OpenFlow v.1.0-centric and contains a Match and an Action.
  */
 
 public class FlowEntry {
+    public static final int PRIORITY_DEFAULT = 32768; // Default Flow Priority
+
     protected long sw;
     protected Match match;
     protected Set<Action> actions;
@@ -37,12 +47,12 @@
      * @param dstIpAddress destination IP address
      * @param operator OpenFlow operation/command (add, remove, etc.)
      */
-// CHECKSTYLE:OFF suppress the warning about too many parameters
+    // CHECKSTYLE:OFF suppress the warning about too many parameters
     public FlowEntry(long sw, long srcPort, long dstPort,
-                     MACAddress srcMac, MACAddress dstMac,
-                     int srcIpAddress, int dstIpAddress,
-                     Operator operator) {
-// CHECKSTYLE:ON
+            MACAddress srcMac, MACAddress dstMac,
+            int srcIpAddress, int dstIpAddress,
+            Operator operator) {
+        // CHECKSTYLE:ON
         this.sw = sw;
         this.match = new Match(sw, srcPort, srcMac, dstMac, srcIpAddress, dstIpAddress);
         this.actions = new HashSet<Action>();
@@ -133,33 +143,91 @@
     }
 
     /**
-     * Converts the FlowEntry in to a legacy FlowEntry object.
+     * Builds and returns an OFFlowMod given an OFFactory.
      *
-     * @return an equivalent legacy FlowEntry object
+     * @param factory the OFFactory to use for building
+     * @return the OFFlowMod
      */
-    public net.onrc.onos.core.util.FlowEntry getFlowEntry() {
-        net.onrc.onos.core.util.FlowEntry entry = new net.onrc.onos.core.util.FlowEntry();
-        entry.setDpid(new Dpid(sw));
-        entry.setFlowEntryId(new FlowEntryId(flowEntryId));
-        entry.setFlowEntryMatch(match.getFlowEntryMatch());
-        FlowEntryActions flowEntryActions = new FlowEntryActions();
-        for (Action action : actions) {
-            flowEntryActions.addAction(action.getFlowEntryAction());
-        }
-        entry.setFlowEntryActions(flowEntryActions);
+    public OFFlowMod buildFlowMod(OFFactory factory) {
+        OFFlowMod.Builder builder = null;
+
         switch (operator) {
-            case ADD:
-                entry.setFlowEntryUserState(FlowEntryUserState.FE_USER_MODIFY);
-                break;
-            case REMOVE:
-                entry.setFlowEntryUserState(FlowEntryUserState.FE_USER_DELETE);
-                break;
-            default:
-                break;
+        case ADD:
+            builder = factory.buildFlowModifyStrict();
+            break;
+        case REMOVE:
+            builder = factory.buildFlowDeleteStrict();
+            break;
+        default:
+            // TODO throw error?
+            return null;
         }
-        entry.setIdleTimeout(idleTimeout);
-        entry.setHardTimeout(hardTimeout);
-        return entry;
+
+        // Build OFMatch
+        Builder matchBuilder = match.getOFMatchBuilder(factory);
+
+        // Build OFAction Set
+        List<OFAction> actionList = new ArrayList<>(actions.size());
+        for (Action action : actions) {
+            actionList.add(action.getOFAction(factory));
+        }
+
+        OFPort outp = OFPort.of((short) 0xffff); // OF1.0 OFPP.NONE
+        if (operator == Operator.REMOVE) {
+            if (actionList.size() == 1) {
+                if (actionList.get(0).getType() == OFActionType.OUTPUT) {
+                    OFActionOutput oa = (OFActionOutput) actionList.get(0);
+                    outp = oa.getPort();
+                }
+            }
+        }
+
+        // Build OFFlowMod
+        builder.setMatch(matchBuilder.build())
+                .setActions(actionList)
+                .setIdleTimeout(idleTimeout)
+                .setHardTimeout(hardTimeout)
+                .setCookie(U64.of(flowEntryId))
+                .setBufferId(OFBufferId.NO_BUFFER)
+                .setPriority(PRIORITY_DEFAULT)
+                .setOutPort(outp);
+
+        /* Note: The following are NOT USED.
+         * builder.setFlags()
+         * builder.setInstructions()
+         * builder.setOutGroup()
+         * builder.setTableId()
+         * builder.setXid()
+         */
+
+        // TODO from Flow Pusher
+        // Set the OFPFF_SEND_FLOW_REM flag if the Flow Entry is not
+        // permanent.
+        //
+        // if ((flowEntry.idleTimeout() != 0) ||
+        // (flowEntry.hardTimeout() != 0)) {
+        // fm.setFlags(OFFlowMod.OFPFF_SEND_FLOW_REM);
+        // }
+
+        // TODO do we care?
+        // fm.setOutPort(OFPort.OFPP_NONE.getValue());
+        // if ((flowModCommand == OFFlowMod.OFPFC_DELETE)
+        // || (flowModCommand == OFFlowMod.OFPFC_DELETE_STRICT)) {
+        // if (actionOutputPort.portNumber != null) {
+        // fm.setOutPort(actionOutputPort.portNumber);
+        // }
+        // }
+
+        // TODO
+        // Set the OFPFF_SEND_FLOW_REM flag if the Flow Entry is not
+        // permanent.
+        //
+        // if ((flowEntry.idleTimeout() != 0) ||
+        // (flowEntry.hardTimeout() != 0)) {
+        // fm.setFlags(OFFlowMod.OFPFF_SEND_FLOW_REM);
+        // }
+
+        return builder.build();
     }
 
     /**
diff --git a/src/main/java/net/onrc/onos/core/intent/ForwardAction.java b/src/main/java/net/onrc/onos/core/intent/ForwardAction.java
index ef6dd00..a3792d1 100644
--- a/src/main/java/net/onrc/onos/core/intent/ForwardAction.java
+++ b/src/main/java/net/onrc/onos/core/intent/ForwardAction.java
@@ -3,6 +3,10 @@
 import net.onrc.onos.core.util.FlowEntryAction;
 import net.onrc.onos.core.util.PortNumber;
 
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.types.OFPort;
+
 /**
  * A class to represent the OpenFlow forwarding action.
  */
@@ -41,6 +45,11 @@
         return action;
     }
 
+    @Override
+    public OFAction getOFAction(OFFactory factory) {
+        return factory.actions().output(OFPort.of((int) dstPort), Short.MAX_VALUE);
+    }
+
     /**
      * A simple hash function that just used the destination port.
      *
diff --git a/src/main/java/net/onrc/onos/core/intent/Match.java b/src/main/java/net/onrc/onos/core/intent/Match.java
index cddd6de..a3c397b 100644
--- a/src/main/java/net/onrc/onos/core/intent/Match.java
+++ b/src/main/java/net/onrc/onos/core/intent/Match.java
@@ -1,6 +1,5 @@
 package net.onrc.onos.core.intent;
 
-
 import java.util.Objects;
 
 import net.floodlightcontroller.util.MACAddress;
@@ -10,6 +9,14 @@
 import net.onrc.onos.core.util.IPv4Net;
 import net.onrc.onos.core.util.PortNumber;
 
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.match.Match.Builder;
+import org.projectfloodlight.openflow.protocol.match.MatchField;
+import org.projectfloodlight.openflow.types.EthType;
+import org.projectfloodlight.openflow.types.IPv4AddressWithMask;
+import org.projectfloodlight.openflow.types.MacAddress;
+import org.projectfloodlight.openflow.types.OFPort;
+
 /**
  * A class to represent the OpenFlow match.
  * <p>
@@ -41,8 +48,10 @@
      * @param dstMac destination Ethernet MAC address
      */
     public Match(long sw, long srcPort,
-                 MACAddress srcMac, MACAddress dstMac) {
-        this(sw, srcPort, srcMac, dstMac, ShortestPathIntent.EMPTYIPADDRESS, ShortestPathIntent.EMPTYIPADDRESS);
+            MACAddress srcMac, MACAddress dstMac) {
+        this(sw, srcPort, srcMac, dstMac,
+                ShortestPathIntent.EMPTYIPADDRESS,
+                ShortestPathIntent.EMPTYIPADDRESS);
     }
 
     /**
@@ -56,7 +65,7 @@
      * @param dstIp destination IP address
      */
     public Match(long sw, long srcPort, MACAddress srcMac, MACAddress dstMac,
-                 int srcIp, int dstIp) {
+            int srcIp, int dstIp) {
 
         this.sw = sw;
         this.srcPort = srcPort;
@@ -75,7 +84,7 @@
     public boolean equals(Object obj) {
         if (obj instanceof Match) {
             Match other = (Match) obj;
-            //TODO: we might consider excluding sw from this comparison
+            // TODO: we might consider excluding sw from this comparison
             if (this.sw != other.sw) {
                 return false;
             }
@@ -130,6 +139,30 @@
         return match;
     }
 
+    public Builder getOFMatchBuilder(OFFactory factory) {
+        Builder matchBuilder = factory.buildMatch();
+
+        if (srcMac != null) {
+            matchBuilder.setExact(MatchField.ETH_SRC, MacAddress.of(srcMac.toLong()));
+        }
+        if (dstMac != null) {
+            matchBuilder.setExact(MatchField.ETH_DST, MacAddress.of(dstMac.toLong()));
+        }
+        if (srcIp != ShortestPathIntent.EMPTYIPADDRESS) {
+            matchBuilder.setExact(MatchField.ETH_TYPE, EthType.IPv4)
+                    .setMasked(MatchField.IPV4_SRC,
+                            IPv4AddressWithMask.of(srcIp, IPV4_PREFIX_LEN));
+        }
+        if (dstIp != ShortestPathIntent.EMPTYIPADDRESS) {
+            matchBuilder.setExact(MatchField.ETH_TYPE, EthType.IPv4)
+                    .setMasked(MatchField.IPV4_DST,
+                            IPv4AddressWithMask.of(dstIp, IPV4_PREFIX_LEN));
+        }
+        matchBuilder.setExact(MatchField.IN_PORT, OFPort.of((int) srcPort));
+
+        return matchBuilder;
+    }
+
     /**
      * Returns a String representation of this Match.
      *
@@ -137,7 +170,9 @@
      */
     @Override
     public String toString() {
-        return "Sw:" + sw + " (" + srcPort + "," + srcMac + "," + dstMac + "," + srcIp + "," + dstIp + ")";
+        return "Sw:" + sw + " (" + srcPort + ","
+                + srcMac + "," + dstMac + ","
+                + srcIp + "," + dstIp + ")";
     }
 
     /**
@@ -147,8 +182,8 @@
      */
     @Override
     public int hashCode() {
-        //TODO: we might consider excluding sw from the hash function
-        //      to make it easier to compare matches between switches
-        return  Objects.hash(sw, srcPort, srcMac, dstMac, srcIp, dstIp);
+        // TODO: we might consider excluding sw from the hash function
+        // to make it easier to compare matches between switches
+        return Objects.hash(sw, srcPort, srcMac, dstMac, srcIp, dstIp);
     }
 }
diff --git a/src/main/java/net/onrc/onos/core/intent/runtime/PlanInstallModule.java b/src/main/java/net/onrc/onos/core/intent/runtime/PlanInstallModule.java
index 025b6d7..058b2d0 100644
--- a/src/main/java/net/onrc/onos/core/intent/runtime/PlanInstallModule.java
+++ b/src/main/java/net/onrc/onos/core/intent/runtime/PlanInstallModule.java
@@ -31,17 +31,17 @@
 import net.onrc.onos.core.intent.ShortestPathIntent;
 import net.onrc.onos.core.topology.ITopologyService;
 
-import org.openflow.protocol.OFFlowRemoved;
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFFlowRemoved;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * The PlanInstallModule contains the PlanCalcRuntime and PlanInstallRuntime.
  * <p>
- * It is responsible for converting Intents into FlowMods and seeing that
- * they are properly installed.
+ * It is responsible for converting Intents into FlowMods and seeing that they
+ * are properly installed.
  */
 public class PlanInstallModule implements IFloodlightModule, IOFMessageListener {
 
@@ -76,8 +76,8 @@
             while (true) {
                 try {
                     IntentOperationList intents = intentQueue.take();
-                    //TODO: consider draining the remaining intent lists
-                    //      and processing in one big batch
+                    // TODO: consider draining the remaining intent lists
+                    // and processing in one big batch
 
                     processIntents(intents);
                 } catch (InterruptedException e) {
@@ -111,8 +111,9 @@
         }
 
         /***
-         * This function is for sending intent state notification to other ONOS instances.
-         * The argument of "domainSwitchDpids" is required for dispatching this ONOS's managed switches.
+         * This function is for sending intent state notification to other ONOS
+         * instances. The argument of "domainSwitchDpids" is required for
+         * dispatching this ONOS's managed switches.
          *
          * @param intents list of intents
          * @param installed true if Intents were installed
@@ -125,31 +126,32 @@
             for (IntentOperation i : intents) {
                 IntentState newState;
                 switch (i.operator) {
-                    case REMOVE:
-                        if (installed) {
-                            newState = success ? IntentState.DEL_ACK : IntentState.DEL_PENDING;
-                        } else {
-                            newState = IntentState.DEL_REQ;
+                case REMOVE:
+                    if (installed) {
+                        newState = success ? IntentState.DEL_ACK
+                                : IntentState.DEL_PENDING;
+                    } else {
+                        newState = IntentState.DEL_REQ;
+                    }
+                    break;
+                case ADD:
+                default:
+                    if (installed) {
+                        if (domainSwitchDpids != null) {
+                            states.domainSwitchDpids.addAll(domainSwitchDpids);
                         }
-                        break;
-                    case ADD:
-                    default:
-                        if (installed) {
-                            if (domainSwitchDpids != null) {
-                                states.domainSwitchDpids.addAll(domainSwitchDpids);
-                            }
-                            newState = success ? IntentState.INST_ACK : IntentState.INST_NACK;
-                        } else {
-                            newState = IntentState.INST_REQ;
-                        }
-                        break;
+                        newState = success ? IntentState.INST_ACK : IntentState.INST_NACK;
+                    } else {
+                        newState = IntentState.INST_REQ;
+                    }
+                    break;
                 }
                 states.put(i.intent.getId(), newState);
             }
 
             if (log.isTraceEnabled()) {
                 log.trace("sendNotifications, states {}, domainSwitchDpids {}",
-                       states, states.domainSwitchDpids);
+                        states, states.domainSwitchDpids);
             }
 
             // Send notifications using the same key every time
@@ -222,6 +224,7 @@
 
     /**
      * Formatted log for debugging.
+     * <p>
      * TODO: merge this into debugging framework
      *
      * @param step the step of computation
@@ -261,7 +264,8 @@
                 IntentOperationList.class);
         eventListener.start();
         // start publisher
-        intentStateChannel = datagridService.createChannel(INTENT_STATE_EVENT_CHANNEL_NAME,
+        intentStateChannel = datagridService.createChannel(
+                INTENT_STATE_EVENT_CHANNEL_NAME,
                 Long.class,
                 IntentStateList.class);
         floodlightProvider.addOFMessageListener(OFType.FLOW_REMOVED, this);
@@ -308,19 +312,19 @@
     @Override
     public Command receive(IOFSwitch sw, OFMessage msg, FloodlightContext cntx) {
         if (msg.getType().equals(OFType.FLOW_REMOVED) &&
-            (msg instanceof OFFlowRemoved)) {
+                (msg instanceof OFFlowRemoved)) {
             OFFlowRemoved flowRemovedMsg = (OFFlowRemoved) msg;
 
             if (log.isTraceEnabled()) {
-               log.trace("Receive flowRemoved from sw {} : Cookie {}",
-                       sw.getId(), flowRemovedMsg.getCookie());
+                log.trace("Receive flowRemoved from sw {} : Cookie {}",
+                        sw.getId(), flowRemovedMsg.getCookie());
             }
 
-            String intentParentId = Long.toString(flowRemovedMsg.getCookie());
+            String intentParentId = Long.toString(flowRemovedMsg.getCookie().getValue());
             Intent intent = parentIntentMap.get(intentParentId);
 
-            //We assume if the path src sw flow entry is expired,
-            //the path is expired.
+            // We assume if the path src sw flow entry is expired,
+            // the path is expired.
             if (!isFlowSrcRemoved(sw.getId(), intentParentId)) {
                 return Command.CONTINUE;
             }
@@ -343,7 +347,8 @@
             log.debug("addEntry to intentStateChannel intentId {}, states {}",
                     pathIntentId, newState);
 
-            intentStateChannel.addTransientEntry(flowRemovedMsg.getCookie(), states);
+            intentStateChannel.addTransientEntry(flowRemovedMsg.getCookie().getValue(),
+                    states);
         }
 
         return Command.CONTINUE;
@@ -355,10 +360,10 @@
      * @param dpid DPID of affected switch
      * @param shortestPathIntentId Intent to check
      * @return true if the flow's source switch entry is removed or expired,
-     * otherwise false.
+     *         otherwise false.
      */
     private boolean isFlowSrcRemoved(long dpid, String shortestPathIntentId) {
-        Intent intent =  parentIntentMap.get(shortestPathIntentId);
+        Intent intent = parentIntentMap.get(shortestPathIntentId);
         ShortestPathIntent spfIntent = null;
         if (intent instanceof ShortestPathIntent) {
             spfIntent = (ShortestPathIntent) intent;
@@ -376,33 +381,19 @@
         return false;
     }
 
-    /*
-     * (non-Javadoc)
-     * @see net.floodlightcontroller.core.IListener#getName()
-     */
     @Override
     public String getName() {
-        // TODO Auto-generated method stub
-        return null;
+        return "planInstall";
     }
 
-    /*
-     * (non-Javadoc)
-     * @see net.floodlightcontroller.core.IListener#isCallbackOrderingPrereq(java.lang.Object, java.lang.String)
-     */
     @Override
     public boolean isCallbackOrderingPrereq(OFType type, String name) {
-        // TODO Auto-generated method stub
         return false;
     }
 
-    /*
-     * (non-Javadoc)
-     * @see net.floodlightcontroller.core.IListener#isCallbackOrderingPostreq(java.lang.Object, java.lang.String)
-     */
     @Override
     public boolean isCallbackOrderingPostreq(OFType type, String name) {
-        // TODO Auto-generated method stub
         return false;
     }
+
 }
diff --git a/src/main/java/net/onrc/onos/core/intent/runtime/PlanInstallRuntime.java b/src/main/java/net/onrc/onos/core/intent/runtime/PlanInstallRuntime.java
index f942fa8..8a444d7 100644
--- a/src/main/java/net/onrc/onos/core/intent/runtime/PlanInstallRuntime.java
+++ b/src/main/java/net/onrc/onos/core/intent/runtime/PlanInstallRuntime.java
@@ -16,14 +16,15 @@
 import net.onrc.onos.core.intent.FlowEntry;
 import net.onrc.onos.core.util.Pair;
 
-import org.openflow.protocol.OFBarrierReply;
+import org.projectfloodlight.openflow.protocol.OFBarrierReply;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * This class is responsible for installing plans (lists of sets of FlowEntries) into local switches.
- * In this context, a local switch is a switch for which this ONOS instance is the master.
- * It also is responsible for sending barrier messages between sets.
+ * This class is responsible for installing plans (lists of sets of FlowEntries)
+ * into local switches. In this context, a local switch is a switch for which
+ * this ONOS instance is the master. It also is responsible for sending barrier
+ * messages between sets.
  */
 
 public class PlanInstallRuntime {
@@ -39,16 +40,18 @@
      * @param pusher the FlowPusherService to use for FlowEntry installation
      */
     public PlanInstallRuntime(IFloodlightProviderService provider,
-                              IFlowPusherService pusher) {
+            IFlowPusherService pusher) {
         this.provider = provider;
         this.pusher = pusher;
     }
 
     /**
-     * This class is a temporary class for collecting FlowMod installation information. It is
-     * largely used for debugging purposes, and it should not be depended on for other purposes.
+     * This class is a temporary class for collecting FlowMod installation
+     * information. It is largely used for debugging purposes, and it should not
+     * be depended on for other purposes.
      * <p>
-     * TODO: This class should be wrapped into a more generic debugging framework when available.
+     * TODO: This class should be wrapped into a more generic debugging
+     * framework when available.
      */
     private static class FlowModCount {
         WeakReference<IOFSwitch> sw;
@@ -72,17 +75,17 @@
          */
         void addFlowEntry(FlowEntry entry) {
             switch (entry.getOperator()) {
-                case ADD:
-                    modFlows++;
-                    break;
-                case ERROR:
-                    errors++;
-                    break;
-                case REMOVE:
-                    delFlows++;
-                    break;
-                default:
-                    break;
+            case ADD:
+                modFlows++;
+                break;
+            case ERROR:
+                errors++;
+                break;
+            case REMOVE:
+                delFlows++;
+                break;
+            default:
+                break;
             }
         }
 
@@ -101,13 +104,15 @@
         static Map<IOFSwitch, FlowModCount> map = new WeakHashMap<>();
 
         /**
-         * This function is used for collecting statistics information. It should be called for
-         * every FlowEntry that is pushed to the switch for accurate statistics.
+         * This function is used for collecting statistics information. It
+         * should be called for every FlowEntry that is pushed to the switch for
+         * accurate statistics.
          * <p>
-         * This class maintains a map of Switches and FlowModCount collection objects, which
-         * are used for collection.
+         * This class maintains a map of Switches and FlowModCount collection
+         * objects, which are used for collection.
          * <p>
-         * TODO: This should be refactored to use a more generic mechanism when available.
+         * TODO: This should be refactored to use a more generic mechanism when
+         * available.
          *
          * @param sw the switch that entry is being pushed to
          * @param entry the FlowEntry being pushed
@@ -122,7 +127,8 @@
         }
 
         /**
-         * Reset the statistics collection. It should be called when required for debugging.
+         * Reset the statistics collection. It should be called when required
+         * for debugging.
          */
         static void startCount() {
             map.clear();
@@ -148,11 +154,11 @@
     /**
      * This function should be called to install the FlowEntries in the plan.
      * <p>
-     * Each set of FlowEntries can be installed together, but all entries should be installed
-     * proceeded to the next set.
+     * Each set of FlowEntries can be installed together, but all entries should
+     * be installed proceeded to the next set.
      * <p>
-     * TODO: This method lack coordination between the other ONOS instances before proceeded
-     * with the next set of entries
+     * TODO: This method lack coordination between the other ONOS instances
+     * before proceeded with the next set of entries
      *
      * @param plan list of set of FlowEntries for installation on local switches
      * @return true (we assume installation is successful)
@@ -164,7 +170,7 @@
         log.debug("IOFSwitches: {}", switches);
         FlowModCount.startCount();
         for (Set<FlowEntry> phase : plan) {
-            Set<Pair<IOFSwitch, net.onrc.onos.core.util.FlowEntry>> entries = new HashSet<>();
+            Set<Pair<IOFSwitch, FlowEntry>> entries = new HashSet<>();
             Set<IOFSwitch> modifiedSwitches = new HashSet<>();
 
             long step1 = System.nanoTime();
@@ -176,7 +182,7 @@
                     log.debug("Skipping flow entry: {}", entry);
                     continue;
                 }
-                entries.add(new Pair<>(sw, entry.getFlowEntry()));
+                entries.add(new Pair<>(sw, entry));
                 modifiedSwitches.add(sw);
                 FlowModCount.countFlowEntry(sw, entry);
             }
diff --git a/src/main/java/net/onrc/onos/core/linkdiscovery/LinkDiscoveryManager.java b/src/main/java/net/onrc/onos/core/linkdiscovery/LinkDiscoveryManager.java
index a924f00..4c87501 100644
--- a/src/main/java/net/onrc/onos/core/linkdiscovery/LinkDiscoveryManager.java
+++ b/src/main/java/net/onrc/onos/core/linkdiscovery/LinkDiscoveryManager.java
@@ -24,7 +24,6 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -38,6 +37,7 @@
 import net.floodlightcontroller.core.IFloodlightProviderService;
 import net.floodlightcontroller.core.IOFMessageListener;
 import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.IOFSwitch.PortChangeType;
 import net.floodlightcontroller.core.IOFSwitchListener;
 import net.floodlightcontroller.core.IUpdate;
 import net.floodlightcontroller.core.annotations.LogMessageCategory;
@@ -57,35 +57,37 @@
 import net.onrc.onos.core.registry.IControllerRegistryService;
 import net.onrc.onos.core.util.SwitchPort;
 
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFPacketIn;
-import org.openflow.protocol.OFPacketOut;
-import org.openflow.protocol.OFPhysicalPort;
-import org.openflow.protocol.OFPhysicalPort.OFPortConfig;
-import org.openflow.protocol.OFPhysicalPort.OFPortState;
-import org.openflow.protocol.OFPort;
-import org.openflow.protocol.OFPortStatus;
-import org.openflow.protocol.OFPortStatus.OFPortReason;
-import org.openflow.protocol.OFType;
-import org.openflow.protocol.action.OFAction;
-import org.openflow.protocol.action.OFActionOutput;
-import org.openflow.protocol.action.OFActionType;
-import org.openflow.util.HexString;
+import org.projectfloodlight.openflow.protocol.OFFactories;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFPacketIn;
+import org.projectfloodlight.openflow.protocol.OFPacketOut;
+import org.projectfloodlight.openflow.protocol.OFPortConfig;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFPortReason;
+import org.projectfloodlight.openflow.protocol.OFPortState;
+import org.projectfloodlight.openflow.protocol.OFPortStatus;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.types.OFBufferId;
+import org.projectfloodlight.openflow.types.OFPort;
+import org.projectfloodlight.openflow.util.HexString;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Discovers links between OpenFlow switches.
- * <p/>
- * Discovery is performed by sending probes (LLDP packets) over the links in
- * the data plane. The LinkDiscoveryManager sends probes periodically on all
- * ports on all connected switches. The probes contain the sending switch's
- * DPID and outgoing port number. LLDP packets that are received (via an
- * OpenFlow packet-in) indicate there is a link between the receiving port and
- * the sending port, which was encoded in the LLDP. When the
- * LinkDiscoveryManager observes a new link, a Link object is created and an
- * event is fired for any event listeners.
- * <p/>
+ * <p>
+ * Discovery is performed by sending probes (LLDP packets) over the links in the
+ * data plane. The LinkDiscoveryManager sends probes periodically on all ports
+ * on all connected switches. The probes contain the sending switch's DPID and
+ * outgoing port number. LLDP packets that are received (via an OpenFlow
+ * packet-in) indicate there is a link between the receiving port and the
+ * sending port, which was encoded in the LLDP. When the LinkDiscoveryManager
+ * observes a new link, a Link object is created and an event is fired for any
+ * event listeners.
+ * </p>
  * Links are removed for one of three reasons:
  * <ul>
  * <li>A probe has not been received on the link for an interval (the timeout
@@ -104,6 +106,9 @@
 
     private static final Logger log =
             LoggerFactory.getLogger(LinkDiscoveryManager.class);
+    // TODO Remove these factories.
+    protected OFFactory factory13 = OFFactories.getFactory(OFVersion.OF_13);
+    protected OFFactory factory10 = OFFactories.getFactory(OFVersion.OF_10);
 
     private IFloodlightProviderService controller;
 
@@ -121,13 +126,16 @@
     // Link discovery task details.
     private SingletonTask discoveryTask;
     private static final int DISCOVERY_TASK_INTERVAL = 1;
-    private static final int LINK_TIMEOUT = 35; // original 35 secs, aggressive 5 secs
-    private static final int LLDP_TO_ALL_INTERVAL = 15; //original 15 seconds, aggressive 2 secs.
+    private static final int LINK_TIMEOUT = 35; // original 35 secs, aggressive
+                                                // 5 secs
+    private static final int LLDP_TO_ALL_INTERVAL = 15; // original 15 seconds,
+                                                        // aggressive 2 secs.
     private long lldpClock = 0;
     // This value is intentionally kept higher than LLDP_TO_ALL_INTERVAL.
     // If we want to identify link failures faster, we could decrease this
     // value to a small number, say 1 or 2 sec.
-    private static final int LLDP_TO_KNOWN_INTERVAL = 20; // LLDP frequency for known links
+    private static final int LLDP_TO_KNOWN_INTERVAL = 20; // LLDP frequency for
+                                                          // known links
 
     private ReentrantReadWriteLock lock;
 
@@ -149,8 +157,8 @@
     /**
      * Listeners are called in the order they were added to the the list.
      */
-    private final List<ILinkDiscoveryListener> linkDiscoveryListeners
-            = new CopyOnWriteArrayList<>();
+    private final List<ILinkDiscoveryListener> linkDiscoveryListeners =
+            new CopyOnWriteArrayList<>();
 
     /**
      * List of ports through which LLDPs are not sent.
@@ -230,8 +238,8 @@
         discover(npt);
     }
 
-    private boolean isLinkDiscoverySuppressed(long sw, short portNumber) {
-        return this.suppressLinkDiscovery.contains(new NodePortTuple(sw, portNumber));
+    private boolean isLinkDiscoverySuppressed(long sw, short p) {
+        return this.suppressLinkDiscovery.contains(new NodePortTuple(sw, p));
     }
 
     private void discoverLinks() {
@@ -239,7 +247,7 @@
         // time out known links.
         timeOutLinks();
 
-        //increment LLDP clock
+        // increment LLDP clock
         lldpClock = (lldpClock + 1) % LLDP_TO_ALL_INTERVAL;
 
         if (lldpClock == 0) {
@@ -271,8 +279,8 @@
     }
 
     /**
-     * Send link discovery message out of a given switch port.
-     * The discovery message is a standard LLDP containing ONOS-specific TLVs.
+     * Send link discovery message out of a given switch port. The discovery
+     * message is a standard LLDP containing ONOS-specific TLVs.
      *
      * @param sw the switch to send on
      * @param port the port to send out
@@ -284,18 +292,18 @@
                     "to the switch.",
             recommendation = LogMessageDoc.CHECK_SWITCH)
     protected void sendDiscoveryMessage(long sw, short port,
-                                        boolean isReverse) {
+            boolean isReverse) {
 
         IOFSwitch iofSwitch = floodlightProvider.getSwitches().get(sw);
         if (iofSwitch == null) {
             return;
         }
 
-        if (port == OFPort.OFPP_LOCAL.getValue()) {
+        if (port == OFPort.LOCAL.getShortPortNumber()) {
             return;
         }
 
-        OFPhysicalPort ofpPort = iofSwitch.getPort(port);
+        OFPortDesc ofpPort = iofSwitch.getPort(port);
 
         if (ofpPort == null) {
             if (log.isTraceEnabled()) {
@@ -314,7 +322,9 @@
                     sw, port);
         }
 
-        OFPacketOut po = createLLDPPacketOut(sw, ofpPort, isReverse);
+        OFFactory factory = (iofSwitch.getOFVersion() == OFVersion.OF_10)
+                ? factory10 : factory13;
+        OFPacketOut po = createLLDPPacketOut(sw, ofpPort, isReverse, factory);
 
         try {
             iofSwitch.write(po, null);
@@ -332,10 +342,11 @@
      * @param dpid the dpid of the outgoing switch
      * @param port the outgoing port
      * @param isReverse whether this is a reverse LLDP or not
+     * @param factory the factory to use to create the message
      * @return Packet_out message with LLDP data
      */
     private OFPacketOut createLLDPPacketOut(long dpid,
-            final OFPhysicalPort port, boolean isReverse) {
+            final OFPortDesc port, boolean isReverse, OFFactory factory) {
         // Set up packets
         // TODO optimize by not creating new packets each time
         OnosLldp lldpPacket = new OnosLldp();
@@ -346,29 +357,25 @@
         ethPacket.setPayload(lldpPacket);
         ethPacket.setPad(true);
 
-        final OFPacketOut packetOut = (OFPacketOut) floodlightProvider.getOFMessageFactory()
-                .getMessage(OFType.PACKET_OUT);
-        packetOut.setBufferId(OFPacketOut.BUFFER_ID_NONE);
-
-        final List<OFAction> actionsList = new LinkedList<OFAction>();
-        final OFActionOutput out = (OFActionOutput) floodlightProvider.getOFMessageFactory()
-                .getAction(OFActionType.OUTPUT);
-        out.setPort(port.getPortNumber());
-        actionsList.add(out);
-        packetOut.setActions(actionsList);
-        final short alen = (short) OFActionOutput.MINIMUM_LENGTH;
-
         lldpPacket.setSwitch(dpid);
-        lldpPacket.setPort(port.getPortNumber());
+        lldpPacket.setPort(port.getPortNo().getShortPortNumber());
         lldpPacket.setReverse(isReverse);
-        ethPacket.setSourceMACAddress(port.getHardwareAddress());
-
+        ethPacket.setSourceMACAddress(port.getHwAddr().getBytes());
         final byte[] lldp = ethPacket.serialize();
-        packetOut.setActionsLength(alen);
-        packetOut.setPacketData(lldp);
-        packetOut
-                .setLength((short) (OFPacketOut.MINIMUM_LENGTH + alen + lldp.length));
-        return packetOut;
+
+        List<OFAction> actions = new ArrayList<OFAction>();
+        actions.add(factory.actions()
+                .buildOutput()
+                .setPort(OFPort.ofShort(port.getPortNo().getShortPortNumber()))
+                .build());
+        OFPacketOut po = factory.buildPacketOut()
+                .setData(lldp)
+                .setBufferId(OFBufferId.NO_BUFFER)
+                .setInPort(OFPort.CONTROLLER)
+                .setActions(actions)
+                .build();
+
+        return po;
     }
 
     /**
@@ -383,12 +390,14 @@
             if (sw.getEnabledPorts() == null) {
                 continue;
             }
-            for (OFPhysicalPort ofp : sw.getEnabledPorts()) {
-                if (isLinkDiscoverySuppressed(sw.getId(), ofp.getPortNumber())) {
+            for (OFPortDesc ofp : sw.getEnabledPorts()) {
+                if (isLinkDiscoverySuppressed(sw.getId(),
+                        ofp.getPortNo().getShortPortNumber())) {
                     continue;
                 }
 
-                sendDiscoveryMessage(sw.getId(), ofp.getPortNumber(), false);
+                sendDiscoveryMessage(sw.getId(),
+                        ofp.getPortNo().getShortPortNumber(), false);
             }
         }
     }
@@ -401,31 +410,31 @@
     @Override
     public Command receive(IOFSwitch sw, OFMessage msg, FloodlightContext cntx) {
         switch (msg.getType()) {
-            case PACKET_IN:
-                if (msg instanceof OFPacketIn) {
-                    return this.handlePacketIn(sw.getId(), (OFPacketIn) msg,
-                                               cntx);
-                }
-                break;
-            case PORT_STATUS:
-                if (msg instanceof OFPortStatus) {
-                    return this.handlePortStatus(sw, (OFPortStatus) msg);
-                }
-                break;
-            default:
-                break;
+        case PACKET_IN:
+            if (msg instanceof OFPacketIn) {
+                return this.handlePacketIn(sw.getId(), (OFPacketIn) msg,
+                        cntx);
+            }
+            break;
+        case PORT_STATUS:
+            if (msg instanceof OFPortStatus) {
+                return this.handlePortStatus(sw, (OFPortStatus) msg);
+            }
+            break;
+        default:
+            break;
         }
         return Command.CONTINUE;
     }
 
-    protected Command handleLldp(LLDP lldp, long sw, OFPacketIn pi) {
+    protected Command handleLldp(LLDP lldp, long sw, OFPacketIn pi, short inport) {
         // If LLDP is suppressed on this port, ignore received packet as well
-        IOFSwitch iofSwitch = floodlightProvider.getSwitches().get(sw);
+        IOFSwitch iofSwitch = floodlightProvider.getSwitch(sw);
         if (iofSwitch == null) {
             return Command.STOP;
         }
 
-        if (isLinkDiscoverySuppressed(sw, pi.getInPort())) {
+        if (isLinkDiscoverySuppressed(sw, inport)) {
             return Command.STOP;
         }
 
@@ -435,7 +444,7 @@
         }
 
         // Verify this LLDP packet matches what we're looking for
-        byte[] packetData = pi.getPacketData();
+        byte[] packetData = pi.getData();
         if (!OnosLldp.isOnosLldp(packetData)) {
             log.trace("Dropping LLDP that wasn't sent by ONOS");
             return Command.STOP;
@@ -444,10 +453,10 @@
         SwitchPort switchPort = OnosLldp.extractSwitchPort(packetData);
         long remoteDpid = switchPort.dpid().value();
         short remotePort = switchPort.port().shortValue();
-        IOFSwitch remoteSwitch = floodlightProvider.getSwitches().get(switchPort.dpid().value());
+        IOFSwitch remoteSwitch = floodlightProvider.getSwitches().get(
+                switchPort.dpid().value());
 
-
-        OFPhysicalPort physicalPort = null;
+        OFPortDesc physicalPort = null;
         if (remoteSwitch != null) {
             physicalPort = remoteSwitch.getPort(remotePort);
             if (!remoteSwitch.portEnabled(remotePort)) {
@@ -466,20 +475,24 @@
                 return Command.STOP;
             }
         }
-        if (!iofSwitch.portEnabled(pi.getInPort())) {
+        if (!iofSwitch.portEnabled(inport)) {
             if (log.isTraceEnabled()) {
                 log.trace("Ignoring link with disabled dest port: " +
-                        "switch {} port {}", sw, pi.getInPort());
+                        "switch {} port {}", sw, inport);
             }
             return Command.STOP;
         }
 
-        int srcPortState = (physicalPort != null) ? physicalPort.getState() : 0;
-        physicalPort = iofSwitch.getPort(pi.getInPort());
-        int dstPortState = (physicalPort != null) ? physicalPort.getState() : 0;
+        // TODO It probably should be empty Set instead of null. Confirm and fix.
+        Set<OFPortState> srcPortState = (physicalPort != null)
+                ? physicalPort.getState() : null;
+        physicalPort = iofSwitch.getPort(inport);
+        Set<OFPortState> dstPortState = (physicalPort != null)
+                ? physicalPort.getState() : null;
 
-        // Store the time of update to this link, and push it out to routingEngine
-        Link lt = new Link(remoteDpid, remotePort, iofSwitch.getId(), pi.getInPort());
+        // Store the time of update to this link, and push it out to
+        // routingEngine
+        Link lt = new Link(remoteDpid, remotePort, iofSwitch.getId(), inport);
 
         LinkInfo linkInfo = new LinkInfo(System.currentTimeMillis(),
                 System.currentTimeMillis(), srcPortState, dstPortState);
@@ -499,8 +512,8 @@
             LinkInfo reverseInfo = links.get(reverseLink);
             if (reverseInfo == null) {
                 // the reverse link does not exist.
-                if (newLinkInfo.getFirstSeenTime() >
-                        System.currentTimeMillis() - LINK_TIMEOUT) {
+                if (newLinkInfo.getFirstSeenTime() > System.currentTimeMillis()
+                        - LINK_TIMEOUT) {
                     this.sendDiscoveryMessage(lt.getDst(), lt.getDstPort(), true);
                 }
             }
@@ -511,13 +524,15 @@
     }
 
     protected Command handlePacketIn(long sw, OFPacketIn pi,
-                                     FloodlightContext cntx) {
+            FloodlightContext cntx) {
         Ethernet eth =
                 IFloodlightProviderService.bcStore.get(cntx,
                         IFloodlightProviderService.CONTEXT_PI_PAYLOAD);
+        short inport = (short) cntx.getStorage()
+                .get(IFloodlightProviderService.CONTEXT_PI_INPORT);
 
         if (eth.getEtherType() == Ethernet.TYPE_LLDP) {
-            return handleLldp((LLDP) eth.getPayload(), sw, pi);
+            return handleLldp((LLDP) eth.getPayload(), sw, pi, inport);
         } else if (eth.getEtherType() < 1500) {
             long destMac = eth.getDestinationMAC().toLong();
             if ((destMac & LINK_LOCAL_MASK) == LINK_LOCAL_VALUE) {
@@ -557,6 +572,7 @@
             // If this is the first time we've seen the link, add the Link
             // object to the data structures/indexes as well
             if (existingInfo == null) {
+                log.trace("Creating new Link: {}", lt);
                 // index it by switch source
                 if (!switchLinks.containsKey(lt.getSrc())) {
                     switchLinks.put(lt.getSrc(), new HashSet<Link>());
@@ -639,8 +655,8 @@
     }
 
     /**
-     * Handles an OFPortStatus message from a switch. We will add or
-     * delete LinkTupes as well re-compute the topology if needed.
+     * Handles an OFPortStatus message from a switch. We will add or delete
+     * LinkTupes as well re-compute the topology if needed.
      *
      * @param sw The dpid of the switch that sent the port status message
      * @param ps The OFPortStatus message
@@ -657,14 +673,14 @@
         if (log.isTraceEnabled()) {
             log.trace("handlePortStatus: Switch {} port #{} reason {}; " +
                     "config is {} state is {}",
-                    new Object[]{sw.getStringId(),
-                            ps.getDesc().getPortNumber(),
+                    new Object[] {sw.getStringId(),
+                            ps.getDesc().getPortNo(),
                             ps.getReason(),
                             ps.getDesc().getConfig(),
                             ps.getDesc().getState()});
         }
 
-        short port = ps.getDesc().getPortNumber();
+        short port = ps.getDesc().getPortNo().getShortPortNumber();
         NodePortTuple npt = new NodePortTuple(sw.getId(), port);
         boolean linkDeleted = false;
         boolean linkInfoChanged = false;
@@ -673,14 +689,12 @@
         try {
             // if ps is a delete, or a modify where the port is down or
             // configured down
-            if ((byte) OFPortReason.OFPPR_DELETE.ordinal() == ps.getReason() ||
-                    ((byte) OFPortReason.OFPPR_MODIFY.ordinal() ==
-                            ps.getReason() && !portEnabled(ps.getDesc()))) {
-
+            if (OFPortReason.DELETE == ps.getReason() ||
+                    (OFPortReason.MODIFY == ps.getReason() &&
+                    !portEnabled(ps.getDesc()))) {
                 deleteLinksOnPort(npt);
                 linkDeleted = true;
-            } else if (ps.getReason() ==
-                    (byte) OFPortReason.OFPPR_MODIFY.ordinal()) {
+            } else if (ps.getReason() == OFPortReason.MODIFY) {
                 // If ps is a port modification and the port state has changed
                 // that affects links in the topology
 
@@ -691,18 +705,22 @@
                         LinkInfo newLinkInfo = null;
 
                         if (lt.isSrcPort(npt) &&
-                                linkInfo.getSrcPortState() != ps.getDesc().getState()) {
-                            // If this port status is for the src port and the port
-                            // state has changed, create a new link info with the new state
+                                !linkInfo.getSrcPortState().equals(
+                                        ps.getDesc().getState())) {
+                            // If this port status is for the src port and the
+                            // port state has changed, create a new link info
+                            // with the new state
 
                             newLinkInfo = new LinkInfo(linkInfo.getFirstSeenTime(),
                                     linkInfo.getLastProbeReceivedTime(),
                                     ps.getDesc().getState(),
                                     linkInfo.getDstPortState());
                         } else if (lt.isDstPort(npt) &&
-                                linkInfo.getDstPortState() != ps.getDesc().getState()) {
-                            // If this port status is for the dst port and the port
-                            // state has changed, create a new link info with the new state
+                                !linkInfo.getDstPortState().equals(
+                                        ps.getDesc().getState())) {
+                            // If this port status is for the dst port and the
+                            // port state has changed, create a new link info
+                            // with the new state
 
                             newLinkInfo = new LinkInfo(linkInfo.getFirstSeenTime(),
                                     linkInfo.getLastProbeReceivedTime(),
@@ -718,13 +736,12 @@
                 }
             }
 
-
             if (!linkDeleted && !linkInfoChanged) {
                 if (log.isTraceEnabled()) {
                     log.trace("handlePortStatus: Switch {} port #{} reason {};" +
                             " no links to update/remove",
-                            new Object[]{HexString.toHexString(sw.getId()),
-                                    ps.getDesc().getPortNumber(),
+                            new Object[] {HexString.toHexString(sw.getId()),
+                                    ps.getDesc().getPortNo(),
                                     ps.getReason()});
                 }
             }
@@ -745,32 +762,37 @@
     }
 
     /**
-     * Process a new port.
-     * If link discovery is disabled on the port, then do nothing.
-     * Otherwise, send LLDP message.
+     * Process a new port. If link discovery is disabled on the port, then do
+     * nothing. Otherwise, send LLDP message.
      *
      * @param sw the dpid of the switch the port is on
      * @param p the number of the port
      */
-    private void processNewPort(long sw, short p) {
-        if (isLinkDiscoverySuppressed(sw, p)) {
+    private void processNewPort(long sw, int p) {
+        if (isLinkDiscoverySuppressed(sw, (short) p)) {
             // Do nothing as link discovery is suppressed.
             return;
         } else {
-            discover(sw, p);
+            discover(sw, (short) p);
         }
     }
 
     /**
-     * We send out LLDP messages when a switch is added to discover the topology.
+     * We send out LLDP messages when a switch is added to discover the
+     * topology.
      *
-     * @param sw The IOFSwitch that connected to the controller
+     * @param swId the datapath Id of the new switch
      */
     @Override
-    public void addedSwitch(IOFSwitch sw) {
+    public void switchActivatedMaster(long swId) {
+        IOFSwitch sw = floodlightProvider.getSwitch(swId);
+        if (sw == null) {
+            log.warn("Added switch not available {} ", swId);
+            return;
+        }
         if (sw.getEnabledPorts() != null) {
-            for (Short p : sw.getEnabledPortNumbers()) {
-                processNewPort(sw.getId(), p);
+            for (Integer p : sw.getEnabledPortNumbers()) {
+                processNewPort(swId, p);
             }
         }
     }
@@ -778,23 +800,27 @@
     /**
      * When a switch disconnects we remove any links from our map and notify.
      *
-     * @param iofSwitch the switch that was removed
+     * @param swId the datapath Id of the switch that was removed
      */
     @Override
-    public void removedSwitch(IOFSwitch iofSwitch) {
+    public void switchDisconnected(long swId) {
+        IOFSwitch sw = floodlightProvider.getSwitch(swId);
+        if (sw == null) {
+            log.warn("Removed switch not available {} ", swId);
+            return;
+        }
         // Update event history
-        long sw = iofSwitch.getId();
 
         List<Link> eraseList = new ArrayList<Link>();
         lock.writeLock().lock();
         try {
-            if (switchLinks.containsKey(sw)) {
+            if (switchLinks.containsKey(swId)) {
                 if (log.isTraceEnabled()) {
                     log.trace("Handle switchRemoved. Switch {}; removing links {}",
-                            HexString.toHexString(sw), switchLinks.get(sw));
+                            HexString.toHexString(swId), switchLinks.get(swId));
                 }
                 // add all tuples with an endpoint on this switch to erase list
-                eraseList.addAll(switchLinks.get(sw));
+                eraseList.addAll(switchLinks.get(swId));
                 deleteLinks(eraseList);
             }
         } finally {
@@ -802,14 +828,34 @@
         }
     }
 
+    @Override
+    public void switchActivatedEqual(long swId) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void switchMasterToEqual(long swId) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void switchEqualToMaster(long swId) {
+        // for now treat as switchActivatedMaster
+        switchActivatedMaster(swId);
+    }
+
     /*
-     * We don't react the port changed notifications here. we listen for
+     * We don't react to port changed notifications here. we listen for
      * OFPortStatus messages directly. Might consider using this notifier
      * instead
      */
     @Override
-    public void switchPortChanged(Long switchId) {
-        // no-op
+    public void switchPortChanged(long swId, OFPortDesc port,
+            PortChangeType changeType) {
+        // TODO Auto-generated method stub
+
     }
 
     /**
@@ -823,7 +869,7 @@
             if (log.isTraceEnabled()) {
                 log.trace("handlePortStatus: Switch {} port #{} " +
                         "removing links {}",
-                        new Object[]{HexString.toHexString(npt.getNodeId()),
+                        new Object[] {HexString.toHexString(npt.getNodeId()),
                                 npt.getPortId(),
                                 this.portLinks.get(npt)});
             }
@@ -833,8 +879,8 @@
     }
 
     /**
-     * Iterates through the list of links and deletes if the
-     * last discovery message reception time exceeds timeout values.
+     * Iterates through the list of links and deletes if the last discovery
+     * message reception time exceeds timeout values.
      */
     protected void timeOutLinks() {
         List<Link> eraseList = new ArrayList<Link>();
@@ -850,7 +896,7 @@
                 LinkInfo info = entry.getValue();
 
                 if ((info.getLastProbeReceivedTime() + (1000L * LINK_TIMEOUT)
-                        < curTime)) {
+                < curTime)) {
                     eraseList.add(entry.getKey());
                 }
             }
@@ -861,14 +907,14 @@
         }
     }
 
-    private boolean portEnabled(OFPhysicalPort port) {
+    private boolean portEnabled(OFPortDesc port) {
         if (port == null) {
             return false;
         }
-        if ((OFPortConfig.OFPPC_PORT_DOWN.getValue() & port.getConfig()) > 0) {
+        if (port.getConfig().contains(OFPortConfig.PORT_DOWN)) {
             return false;
         }
-        if ((OFPortState.OFPPS_LINK_DOWN.getValue() & port.getState()) > 0) {
+        if (port.getState().contains(OFPortState.LINK_DOWN)) {
             return false;
         }
         return true;
@@ -955,7 +1001,8 @@
     @LogMessageDocs({
             @LogMessageDoc(level = "ERROR",
                     message = "No storage source found.",
-                    explanation = "Storage source was not initialized; cannot initialize " +
+                    explanation = "Storage source was not initialized; cannot initialize "
+                            +
                             "link discovery.",
                     recommendation = LogMessageDoc.REPORT_CONTROLLER_BUG),
             @LogMessageDoc(level = "ERROR",
@@ -966,7 +1013,8 @@
                     recommendation = LogMessageDoc.REPORT_CONTROLLER_BUG),
             @LogMessageDoc(level = "ERROR",
                     message = "No storage source found.",
-                    explanation = "Storage source was not initialized; cannot initialize " +
+                    explanation = "Storage source was not initialized; cannot initialize "
+                            +
                             "link discovery.",
                     recommendation = LogMessageDoc.REPORT_CONTROLLER_BUG),
             @LogMessageDoc(level = "ERROR",
diff --git a/src/main/java/net/onrc/onos/core/linkdiscovery/LinkInfo.java b/src/main/java/net/onrc/onos/core/linkdiscovery/LinkInfo.java
index fc19951..bc11c91 100644
--- a/src/main/java/net/onrc/onos/core/linkdiscovery/LinkInfo.java
+++ b/src/main/java/net/onrc/onos/core/linkdiscovery/LinkInfo.java
@@ -1,6 +1,7 @@
 /**
  *    Copyright 2011, Big Switch Networks, Inc.
  *    Originally created by David Erickson, Stanford University
+ *
  *    Licensed under the Apache License, Version 2.0 (the "License"); you may
  *    not use this file except in compliance with the License. You may obtain
  *    a copy of the License at
@@ -12,34 +13,35 @@
  *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  *    License for the specific language governing permissions and limitations
  *    under the License.
- */
+ **/
 
 package net.onrc.onos.core.linkdiscovery;
 
+import java.util.Set;
+
 import net.onrc.onos.core.linkdiscovery.ILinkDiscoveryService.LinkType;
 
+import org.projectfloodlight.openflow.protocol.OFPortState;
+
 import com.google.common.primitives.Longs;
 
 /**
  * Records information about a link.
  */
 public final class LinkInfo {
-
     /**
-     * The port states stored here are topology's last knowledge of
-     * the state of the port. This mostly mirrors the state
-     * maintained in the port list in IOFSwitch (i.e. the one returned
-     * from getPort), except that during a port status message the
-     * IOFSwitch port state will already have been updated with the
-     * new port state, so topology needs to keep its own copy so that
-     * it can determine if the port state has changed and therefore
-     * requires the new state to be written to storage.
-     *
-     * Note the port state values are defined in the OF 1.0 spec.
-     * These will change in some way once we move to OF 1.3.
+     * The port states stored here are topology's last knowledge of the state of
+     * the port. This mostly mirrors the state maintained in the port list in
+     * IOFSwitch (i.e. the one returned from getPort), except that during a port
+     * status message the IOFSwitch port state will already have been updated
+     * with the new port state, so topology needs to keep its own copy so that
+     * it can determine if the port state has changed and therefore requires the
+     * new state to be written to storage. Note the port state values are
+     * defined in the OF 1.0 spec. These will change in some way once we move to
+     * OF 1.3.
      */
-    private final int srcPortState;
-    private final int dstPortState;
+    private final Set<OFPortState> srcPortState;
+    private final Set<OFPortState> dstPortState;
 
     private final long firstSeenTime;
     private final long lastLldpReceivedTime;
@@ -54,8 +56,8 @@
      */
     public LinkInfo(long firstSeenTime,
             long lastLldpReceivedTime,
-            int srcPortState,
-            int dstPortState) {
+            Set<OFPortState> srcPortState,
+            Set<OFPortState> dstPortState) {
         this.srcPortState = srcPortState;
         this.dstPortState = dstPortState;
         this.firstSeenTime = firstSeenTime;
@@ -85,20 +87,47 @@
      *
      * @return the source port state, as defined in the OF1.0 spec
      */
-    public int getSrcPortState() {
+    public Set<OFPortState> getSrcPortState() {
         return srcPortState;
     }
 
+    public int getSrcPortStateInteger() {
+        return convertPortState(srcPortState);
+    }
+
     /**
      * Gets the state of the destination port.
      *
      * @return the destination port state, as defined in the OF1.0 spec
      */
-    public int getDstPortState() {
+    public Set<OFPortState> getDstPortState() {
         return dstPortState;
     }
 
     /**
+     * Gets the state of the destination port.
+     *
+     * @return the destination port state, as defined in the OF1.0 spec
+     */
+    public int getDstPortStateInteger() {
+        return convertPortState(dstPortState);
+    }
+
+    private int convertPortState(Set<OFPortState> ps) {
+        int ret = 0;
+        if (ps.contains(OFPortState.LINK_DOWN)) {
+            ret = 1 << 0;
+        }
+        if (ps.contains(OFPortState.BLOCKED)) {
+            ret = ret | 1 << 1;
+        }
+        if (ps.contains(OFPortState.LIVE)) {
+            ret = ret | 1 << 2;
+        }
+        return ret;
+    }
+
+    /**
      * Gets the link type.
      *
      * @return the link type
@@ -117,8 +146,8 @@
         int result = 1;
         result = prime * result + Longs.hashCode(firstSeenTime);
         result = prime * result + Longs.hashCode(lastLldpReceivedTime);
-        result = prime * result + srcPortState;
-        result = prime * result + dstPortState;
+        result = prime * result + convertPortState(srcPortState);
+        result = prime * result + convertPortState(dstPortState);
         return result;
     }
 
@@ -138,12 +167,11 @@
         LinkInfo other = (LinkInfo) obj;
 
         return firstSeenTime == other.firstSeenTime &&
-               lastLldpReceivedTime == other.lastLldpReceivedTime &&
-               srcPortState == other.srcPortState &&
-               dstPortState == other.dstPortState;
+                lastLldpReceivedTime == other.lastLldpReceivedTime &&
+                srcPortState == other.srcPortState &&
+                dstPortState == other.dstPortState;
     }
 
-
     /* (non-Javadoc)
      * @see java.lang.Object#toString()
      */
diff --git a/src/main/java/net/onrc/onos/core/linkdiscovery/web/LinksResource.java b/src/main/java/net/onrc/onos/core/linkdiscovery/web/LinksResource.java
index 2738041..cb83972 100644
--- a/src/main/java/net/onrc/onos/core/linkdiscovery/web/LinksResource.java
+++ b/src/main/java/net/onrc/onos/core/linkdiscovery/web/LinksResource.java
@@ -28,8 +28,8 @@
                 Link link = e.getKey();
                 LinkInfo info = e.getValue();
                 LinkWithType lwt = new LinkWithType(link,
-                        info.getSrcPortState(),
-                        info.getDstPortState(),
+                        info.getSrcPortStateInteger(),
+                        info.getDstPortStateInteger(),
                         info.getLinkType());
                 returnLinkSet.add(lwt);
             }
diff --git a/src/main/java/net/onrc/onos/core/main/IOFSwitchPortListener.java b/src/main/java/net/onrc/onos/core/main/IOFSwitchPortListener.java
index edf4539..6f6e21b 100644
--- a/src/main/java/net/onrc/onos/core/main/IOFSwitchPortListener.java
+++ b/src/main/java/net/onrc/onos/core/main/IOFSwitchPortListener.java
@@ -3,9 +3,9 @@
  */
 package net.onrc.onos.core.main;
 
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
 import net.floodlightcontroller.core.IOFSwitchListener;
 
-import org.openflow.protocol.OFPhysicalPort;
 
 /**
  * Extra event handler added to IOFSwitchListener by ONOS.
@@ -15,11 +15,11 @@
     /**
      * Fired when ports on a switch area added.
      */
-    public void switchPortAdded(Long switchId, OFPhysicalPort port);
+    public void switchPortAdded(Long switchId, OFPortDesc port);
 
     /**
      * Fired when ports on a switch area removed.
      */
-    public void switchPortRemoved(Long switchId, OFPhysicalPort port);
+    public void switchPortRemoved(Long switchId, OFPortDesc port);
 
 }
diff --git a/src/main/java/net/onrc/onos/core/main/Main.java b/src/main/java/net/onrc/onos/core/main/Main.java
index 78dad3c..7c7ec03 100644
--- a/src/main/java/net/onrc/onos/core/main/Main.java
+++ b/src/main/java/net/onrc/onos/core/main/Main.java
@@ -12,8 +12,6 @@
 
 /**
  * Host for the ONOS main method.
- * <!-- CHECKSTYLE IGNORE WriteTag FOR NEXT 1 LINES -->
- * @author alexreimers
  */
 public final class Main {
 
diff --git a/src/main/java/net/onrc/onos/core/packetservice/PacketModule.java b/src/main/java/net/onrc/onos/core/packetservice/PacketModule.java
index afe2770..63143cd 100644
--- a/src/main/java/net/onrc/onos/core/packetservice/PacketModule.java
+++ b/src/main/java/net/onrc/onos/core/packetservice/PacketModule.java
@@ -30,14 +30,14 @@
 import net.onrc.onos.core.util.PortNumber;
 import net.onrc.onos.core.util.SwitchPort;
 
-import org.openflow.protocol.OFMessage;
-import org.openflow.protocol.OFPacketIn;
-import org.openflow.protocol.OFPacketOut;
-import org.openflow.protocol.OFPhysicalPort;
-import org.openflow.protocol.OFPort;
-import org.openflow.protocol.OFType;
-import org.openflow.protocol.action.OFAction;
-import org.openflow.protocol.action.OFActionOutput;
+import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFPacketIn;
+import org.projectfloodlight.openflow.protocol.OFPacketOut;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.types.OFPort;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,7 +45,7 @@
 import com.google.common.collect.Multimap;
 
 public class PacketModule implements IOFMessageListener, IPacketService,
-                                     IFloodlightModule {
+        IFloodlightModule {
     private static final Logger log = LoggerFactory.getLogger(PacketModule.class);
 
     private final CopyOnWriteArrayList<IPacketListener> listeners;
@@ -54,9 +54,9 @@
     private Topology topology;
     private IDatagridService datagrid;
     private IFlowPusherService flowPusher;
+    private OFFactory factory;
 
-    private IEventChannel<Long, PacketOutNotification>
-            packetOutEventChannel;
+    private IEventChannel<Long, PacketOutNotification> packetOutEventChannel;
 
     private static final String PACKET_OUT_CHANNEL_NAME =
             "onos.packet_out";
@@ -71,8 +71,9 @@
         public void entryAdded(PacketOutNotification value) {
             Multimap<Long, Short> localPorts = HashMultimap.create();
             for (IOFSwitch sw : floodlightProvider.getSwitches().values()) {
-                for (OFPhysicalPort port : sw.getEnabledPorts()) {
-                    localPorts.put(sw.getId(), port.getPortNumber());
+                for (OFPortDesc port : sw.getEnabledPorts()) {
+                    // XXX S fix this to int
+                    localPorts.put(sw.getId(), port.getPortNo().getShortPortNumber());
                 }
             }
             Multimap<Long, Short> outPorts = value.calculateOutPorts(
@@ -104,7 +105,7 @@
     public void sendPacket(Ethernet eth, SwitchPort switchPort) {
         SinglePacketOutNotification notification =
                 new SinglePacketOutNotification(eth.serialize(), 0,
-                switchPort.dpid().value(), switchPort.port().shortValue());
+                        switchPort.dpid().value(), switchPort.port().shortValue());
 
         // TODO We shouldn't care what the destination MAC is
         long dstMac = eth.getDestinationMAC().toLong();
@@ -127,7 +128,7 @@
     public void broadcastPacketOutEdge(Ethernet eth, SwitchPort inSwitchPort) {
         BroadcastPacketOutNotification notification =
                 new BroadcastPacketOutNotification(eth.serialize(), 0,
-                inSwitchPort.dpid().value(), inSwitchPort.port().shortValue());
+                        inSwitchPort.dpid().value(), inSwitchPort.port().shortValue());
 
         long dstMac = eth.getDestinationMAC().toLong();
         packetOutEventChannel.addTransientEntry(dstMac, notification);
@@ -140,7 +141,6 @@
 
     @Override
     public boolean isCallbackOrderingPrereq(OFType type, String name) {
-        // TODO Auto-generated method stub
         return false;
     }
 
@@ -156,18 +156,19 @@
             return Command.CONTINUE;
         }
 
-        OFPacketIn pi = (OFPacketIn) msg;
-
         Ethernet eth = IFloodlightProviderService.bcStore.
                 get(cntx, IFloodlightProviderService.CONTEXT_PI_PAYLOAD);
+        short inport = (short) cntx.getStorage()
+                .get(IFloodlightProviderService.CONTEXT_PI_INPORT);
 
         Switch topologySwitch;
         Port inPort;
-        final Dpid dpid = new Dpid(sw.getId());
-        topology.acquireReadLock();
         try {
+            topology.acquireReadLock();
+            Dpid dpid = new Dpid(sw.getId());
+            PortNumber p = new PortNumber(inport);
             topologySwitch = topology.getSwitch(dpid);
-            inPort = topology.getPort(dpid, new PortNumber(pi.getInPort()));
+            inPort = topology.getPort(dpid, p);
         } finally {
             topology.releaseReadLock();
         }
@@ -194,9 +195,9 @@
 
     @Override
     public Map<Class<? extends IFloodlightService>, IFloodlightService>
-    getServiceImpls() {
-        Map<Class<? extends IFloodlightService>, IFloodlightService>
-        serviceImpls = new HashMap<>();
+            getServiceImpls() {
+
+        Map<Class<? extends IFloodlightService>, IFloodlightService> serviceImpls = new HashMap<>();
         serviceImpls.put(IPacketService.class, this);
         return serviceImpls;
     }
@@ -220,12 +221,12 @@
                 .getTopology();
         datagrid = context.getServiceImpl(IDatagridService.class);
         flowPusher = context.getServiceImpl(IFlowPusherService.class);
+        factory = floodlightProvider.getOFMessageFactory_10();
     }
 
     @Override
     public void startUp(FloodlightModuleContext context) {
         floodlightProvider.addOFMessageListener(OFType.PACKET_IN, this);
-
         packetOutEventChannel = datagrid.addListener(PACKET_OUT_CHANNEL_NAME,
                 packetOutEventHandler,
                 Long.class,
@@ -235,31 +236,25 @@
     private void sendPacketToSwitches(Multimap<Long, Short> outPorts,
             byte[] packetData) {
         for (Long dpid : outPorts.keySet()) {
-            OFPacketOut po = new OFPacketOut();
-            po.setInPort(OFPort.OFPP_NONE)
-              .setBufferId(OFPacketOut.BUFFER_ID_NONE)
-              .setPacketData(packetData);
-
-            List<OFAction> actions = new ArrayList<OFAction>();
-            for (Short port : outPorts.get(dpid)) {
-                actions.add(new OFActionOutput(port));
-            }
-
-            po.setActions(actions);
-            short actionsLength = (short)
-                    (actions.size() * OFActionOutput.MINIMUM_LENGTH);
-            po.setActionsLength(actionsLength);
-            po.setLengthU(OFPacketOut.MINIMUM_LENGTH + actionsLength
-                    + packetData.length);
-
             IOFSwitch sw = floodlightProvider.getSwitches().get(dpid);
 
             if (sw == null) {
-                log.warn("Switch not found when sending packet");
-                return;
+                log.warn("Switch {} not found when sending packet", dpid);
+                continue;
             }
 
+            List<OFAction> actions = new ArrayList<>();
+            for (Short port : outPorts.get(dpid)) {
+                actions.add(factory.actions().output(OFPort.of(port), Short.MAX_VALUE));
+            }
+
+            OFPacketOut po = factory.buildPacketOut()
+                    .setData(packetData)
+                    .setActions(actions)
+                    .build();
+
             flowPusher.add(sw, po);
         }
     }
+
 }
diff --git a/src/main/java/net/onrc/onos/core/registry/IControllerRegistryService.java b/src/main/java/net/onrc/onos/core/registry/IControllerRegistryService.java
index cfc47c8..ef441db 100644
--- a/src/main/java/net/onrc/onos/core/registry/IControllerRegistryService.java
+++ b/src/main/java/net/onrc/onos/core/registry/IControllerRegistryService.java
@@ -8,18 +8,18 @@
 import net.onrc.onos.core.util.OnosInstanceId;
 
 /**
- * A registry service that allows ONOS to register controllers and switches
- * in a way that is global to the entire ONOS cluster. The registry is the
- * arbiter for allowing controllers to control switches.
+ * A registry service that allows ONOS to register controllers and switches in a
+ * way that is global to the entire ONOS cluster. The registry is the arbiter
+ * for allowing controllers to control switches.
  * <p/>
  * The OVS/OF1.{2,3} fault tolerance model is a switch connects to multiple
- * controllers, and the controllers send role requests to determine what their
- * role is in controlling the switch.
+ * controllers, and the controllers send role requests to tell the switch their
+ * role in controlling the switch.
  * <p/>
  * The ONOS fault tolerance model allows only a single controller to have
  * control of a switch (MASTER role) at once. Controllers therefore need a
- * mechanism that enables them to decide who should control a each switch.
- * The registry service provides this mechanism.
+ * mechanism that enables them to decide who should control a each switch. The
+ * registry service provides this mechanism.
  */
 public interface IControllerRegistryService extends IFloodlightService {
 
@@ -29,43 +29,41 @@
     public interface ControlChangeCallback {
         /**
          * Called whenever the control changes from the point of view of the
-         * registry. The callee can check whether they have control or not
-         * using the hasControl parameter.
+         * registry. The callee can check whether they have control or not using
+         * the hasControl parameter.
          *
-         * @param dpid       The switch that control has changed for
+         * @param dpid The switch that control has changed for
          * @param hasControl Whether the listener now has control or not
          */
         void controlChanged(long dpid, boolean hasControl);
     }
 
     /**
-     * Request for control of a switch. This method does not block. When
-     * control for a switch changes, the controlChanged method on the
-     * callback object will be called. This happens any time the control
-     * changes while the request is still active (until releaseControl is
-     * called)
+     * Request for control of a switch. This method does not block. When control
+     * for a switch changes, the controlChanged method on the callback object
+     * will be called. This happens any time the control changes while the
+     * request is still active (until releaseControl is called)
      *
      * @param dpid Switch to request control for
-     * @param cb   Callback that will be used to notify caller of control
-     *             changes
+     * @param cb Callback that will be used to notify caller of control changes
      * @throws RegistryException Errors contacting the registry service
      */
     public void requestControl(long dpid, ControlChangeCallback cb)
             throws RegistryException;
 
     /**
-     * Stop trying to take control of a switch. This removes the entry
-     * for this controller requesting this switch in the registry.
-     * If the controller had control when this is called, another controller
-     * will now gain control of the switch. This call doesn't block.
+     * Stop trying to take control of a switch. This removes the entry for this
+     * controller requesting this switch in the registry. If the controller had
+     * control when this is called, another controller will now gain control of
+     * the switch. This call doesn't block.
      *
      * @param dpid Switch to release control of
      */
     public void releaseControl(long dpid);
 
     /**
-     * Check whether the controller has control of the switch
-     * This call doesn't block.
+     * Check whether the controller has control of the switch This call doesn't
+     * block.
      *
      * @param dpid Switch to check control of
      * @return true if controller has control of the switch.
@@ -73,11 +71,11 @@
     public boolean hasControl(long dpid);
 
     /**
-     * Check whether this instance is the leader for the cluster.
-     * This call doesn't block.
+     * Check whether this instance is the leader for the cluster. This call
+     * doesn't block.
      *
-     * @return true if the instance is the leader for the cluster,
-     * otherwise false.
+     * @return true if the instance is the leader for the cluster, otherwise
+     *         false.
      */
     public boolean isClusterLeader();
 
@@ -89,13 +87,13 @@
     public OnosInstanceId getOnosInstanceId();
 
     /**
-     * Register a controller to the ONOS cluster. Must be called before
-     * the registry can be used to take control of any switches.
+     * Register a controller to the ONOS cluster. Must be called before the
+     * registry can be used to take control of any switches.
      *
-     * @param controllerId A unique string ID identifying this controller
-     *                     in the cluster
+     * @param controllerId A unique string ID identifying this controller in the
+     *        cluster
      * @throws RegistryException for errors connecting to registry service,
-     *                           controllerId already registered
+     *         controllerId already registered
      */
     public void registerController(String controllerId)
             throws RegistryException;
@@ -109,9 +107,9 @@
     public Collection<String> getAllControllers() throws RegistryException;
 
     /**
-     * Get all switches in the cluster, along with which controller is
-     * in control of them (if any) and any other controllers that have
-     * requested control.
+     * Get all switches in the cluster, along with which controller is in
+     * control of them (if any) and any other controllers that have requested
+     * control.
      *
      * @return Map of all switches.
      */
@@ -149,7 +147,6 @@
      */
     public IdBlock allocateUniqueIdBlock(long range);
 
-
     /**
      * Get a globally unique ID.
      *
diff --git a/src/main/java/net/onrc/onos/core/topology/TopologyPublisher.java b/src/main/java/net/onrc/onos/core/topology/TopologyPublisher.java
index ed450f2..8b77745 100644
--- a/src/main/java/net/onrc/onos/core/topology/TopologyPublisher.java
+++ b/src/main/java/net/onrc/onos/core/topology/TopologyPublisher.java
@@ -9,20 +9,20 @@
 import net.floodlightcontroller.core.IFloodlightProviderService;
 import net.floodlightcontroller.core.IFloodlightProviderService.Role;
 import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.IOFSwitch.PortChangeType;
+import net.floodlightcontroller.core.IOFSwitchListener;
 import net.floodlightcontroller.core.module.FloodlightModuleContext;
 import net.floodlightcontroller.core.module.FloodlightModuleException;
 import net.floodlightcontroller.core.module.IFloodlightModule;
 import net.floodlightcontroller.core.module.IFloodlightService;
 import net.floodlightcontroller.core.util.SingletonTask;
 import net.floodlightcontroller.threadpool.IThreadPoolService;
-import net.onrc.onos.api.registry.ILocalSwitchMastershipListener;
 import net.onrc.onos.core.hostmanager.Host;
 import net.onrc.onos.core.hostmanager.IHostListener;
 import net.onrc.onos.core.hostmanager.IHostService;
 import net.onrc.onos.core.linkdiscovery.ILinkDiscoveryListener;
 import net.onrc.onos.core.linkdiscovery.ILinkDiscoveryService;
 import net.onrc.onos.core.linkdiscovery.Link;
-import net.onrc.onos.core.main.IOFSwitchPortListener;
 import net.onrc.onos.core.registry.IControllerRegistryService;
 import net.onrc.onos.core.registry.IControllerRegistryService.ControlChangeCallback;
 import net.onrc.onos.core.registry.RegistryException;
@@ -30,8 +30,8 @@
 import net.onrc.onos.core.util.PortNumber;
 import net.onrc.onos.core.util.SwitchPort;
 
-import org.openflow.protocol.OFPhysicalPort;
-import org.openflow.util.HexString;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.util.HexString;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,12 +40,10 @@
  * discovery modules. These events are reformatted and relayed to the in-memory
  * topology instance.
  */
-public class TopologyPublisher implements /*IOFSwitchListener,*/
-        IOFSwitchPortListener,
+public class TopologyPublisher implements IOFSwitchListener,
         ILinkDiscoveryListener,
         IFloodlightModule,
-        IHostListener,
-        ILocalSwitchMastershipListener {
+        IHostListener {
     private static final Logger log =
             LoggerFactory.getLogger(TopologyPublisher.class);
 
@@ -65,8 +63,8 @@
     private SingletonTask cleanupTask;
 
     /**
-     * Cleanup old switches from the topology. Old switches are those
-     * which have no controller in the registry.
+     * Cleanup old switches from the topology. Old switches are those which have
+     * no controller in the registry.
      */
     private class SwitchCleanup implements ControlChangeCallback, Runnable {
         @Override
@@ -97,7 +95,8 @@
             if (log.isTraceEnabled()) {
                 log.trace("Checking for inactive switches");
             }
-            // For each switch check if a controller exists in controller registry
+            // For each switch check if a controller exists in controller
+            // registry
             for (Switch sw : switches) {
                 // FIXME How to handle case where Switch has never been
                 // registered to ZK
@@ -120,10 +119,10 @@
 
         /**
          * Second half of the switch cleanup operation. If the registry grants
-         * control of a switch, we can be sure no other instance is writing
-         * this switch to the topology, so we can remove it now.
-         *
-         * @param dpid       the dpid of the switch we requested control for
+         * control of a switch, we can be sure no other instance is writing this
+         * switch to the topology, so we can remove it now.
+         * <p>
+         * @param dpid the dpid of the switch we requested control for
          * @param hasControl whether we got control or not
          */
         @Override
@@ -150,7 +149,7 @@
         // TODO define attr name as constant somewhere.
         // TODO populate appropriate attributes.
         linkEvent.createStringAttribute(TopologyElement.TYPE,
-                                        TopologyElement.TYPE_PACKET_LAYER);
+                TopologyElement.TYPE_PACKET_LAYER);
         linkEvent.createStringAttribute(TopologyElement.ELEMENT_CONFIG_STATE,
                 ConfigState.NOT_CONFIGURED.toString());
         linkEvent.createStringAttribute(TopologyElement.ELEMENT_ADMIN_STATUS,
@@ -177,51 +176,173 @@
         // TODO define attr name as constant somewhere.
         // TODO populate appropriate attributes.
         linkEvent.createStringAttribute(TopologyElement.TYPE,
-                                        TopologyElement.TYPE_PACKET_LAYER);
+                TopologyElement.TYPE_PACKET_LAYER);
         linkEvent.freeze();
 
         if (!registryService.hasControl(link.getDst())) {
             // Don't process or send a link event if we're not master for the
             // destination switch
-            log.debug("Not the master for dst switch {}. Suppressed link remove event {}.",
+            log.debug(
+                    "Not the master for dst switch {}. Suppressed link remove event {}.",
                     link.getDst(), linkEvent);
             return;
         }
         topologyDiscoveryInterface.removeLinkDiscoveryEvent(linkEvent);
     }
 
+    /* *****************
+     * IOFSwitchListener
+     * *****************/
+
     @Override
-    public void switchPortAdded(Long switchId, OFPhysicalPort port) {
+    public void switchActivatedMaster(long swId) {
+        IOFSwitch sw = floodlightProvider.getSwitch(swId);
+        final Dpid dpid = new Dpid(swId);
+        if (sw == null) {
+            log.warn("Added switch not available {} ", dpid);
+            return;
+        }
+
+        controllerRoleChanged(dpid, Role.MASTER);
+
+        SwitchEvent switchEvent = new SwitchEvent(dpid);
+        // FIXME should be merging, with existing attrs, etc..
+        // TODO define attr name as constant somewhere.
+        // TODO populate appropriate attributes.
+        switchEvent.createStringAttribute(TopologyElement.TYPE,
+                TopologyElement.TYPE_PACKET_LAYER);
+        switchEvent.createStringAttribute("ConnectedSince",
+                sw.getConnectedSince().toString());
+        switchEvent.createStringAttribute(TopologyElement.ELEMENT_CONFIG_STATE,
+                ConfigState.NOT_CONFIGURED.toString());
+        switchEvent.createStringAttribute(TopologyElement.ELEMENT_ADMIN_STATUS,
+                AdminStatus.ACTIVE.toString());
+        switchEvent.freeze();
+        // TODO Not very robust
+        if (!registryService.hasControl(swId)) {
+            log.debug("Not the master for switch {}. Suppressed switch add event {}.",
+                    dpid, switchEvent);
+            return;
+        }
+        List<PortEvent> portEvents = new ArrayList<PortEvent>();
+        for (OFPortDesc port : sw.getPorts()) {
+            PortEvent portEvent = new PortEvent(dpid,
+                    new PortNumber(port.getPortNo().getShortPortNumber()));
+            // FIXME should be merging, with existing attrs, etc..
+            // TODO define attr name as constant somewhere.
+            // TODO populate appropriate attributes.
+            portEvent.createStringAttribute("name", port.getName());
+            portEvent.createStringAttribute(TopologyElement.TYPE,
+                    TopologyElement.TYPE_PACKET_LAYER);
+            portEvent.createStringAttribute(TopologyElement.ELEMENT_CONFIG_STATE,
+                    ConfigState.NOT_CONFIGURED.toString());
+            portEvent.createStringAttribute(TopologyElement.ELEMENT_ADMIN_STATUS,
+                    AdminStatus.ACTIVE.toString());
+
+            portEvent.freeze();
+            portEvents.add(portEvent);
+        }
+        topologyDiscoveryInterface.putSwitchDiscoveryEvent(switchEvent, portEvents);
+
+        for (OFPortDesc port : sw.getPorts()) {
+            // Allow links to be discovered on this port now that it's
+            // in the database
+            linkDiscovery.enableDiscoveryOnPort(sw.getId(),
+                    port.getPortNo().getShortPortNumber());
+        }
+    }
+
+    @Override
+    public void switchActivatedEqual(long swId) {
+        final Dpid dpid = new Dpid(swId);
+        controllerRoleChanged(dpid, Role.EQUAL);
+    }
+
+    @Override
+    public void switchMasterToEqual(long swId) {
+        final Dpid dpid = new Dpid(swId);
+        controllerRoleChanged(dpid, Role.EQUAL);
+    }
+
+    @Override
+    public void switchEqualToMaster(long swId) {
+        // for now treat as switchActivatedMaster
+        switchActivatedMaster(swId);
+    }
+
+    @Override
+    public void switchDisconnected(long swId) {
+        final Dpid dpid = new Dpid(swId);
+
+        log.debug("Local switch disconnected: dpid = {} role = {}", dpid);
+
+        Role role = Role.SLAVE; // TODO: Should be Role.UNKNOWN
+
+        MastershipEvent mastershipEvent =
+                new MastershipEvent(dpid, registryService.getOnosInstanceId(),
+                        role);
+        // FIXME should be merging, with existing attrs, etc..
+        // TODO define attr name as constant somewhere.
+        // TODO populate appropriate attributes.
+        mastershipEvent.createStringAttribute(TopologyElement.TYPE,
+                TopologyElement.TYPE_ALL_LAYERS);
+        mastershipEvent.freeze();
+        topologyDiscoveryInterface.removeSwitchMastershipEvent(mastershipEvent);
+    }
+
+    @Override
+    public void switchPortChanged(long swId, OFPortDesc port,
+            PortChangeType changeType) {
+        switch (changeType) {
+        case ADD:
+            switchPortAdded(swId, port);
+            break;
+        case DELETE:
+            switchPortRemoved(swId, port);
+            break;
+        case DOWN:
+        case UP:
+        case OTHER_UPDATE:
+        default:
+            // XXX S what is the right set of port change handlers?
+            log.debug("Topology publisher does not handle these port updates: {}",
+                        changeType);
+        }
+    }
+
+    private void switchPortAdded(long switchId, OFPortDesc port) {
         final Dpid dpid = new Dpid(switchId);
-        PortEvent portEvent = new PortEvent(dpid, new PortNumber(port.getPortNumber()));
+        PortEvent portEvent = new PortEvent(dpid,
+                new PortNumber(port.getPortNo().getShortPortNumber()));
         // FIXME should be merging, with existing attrs, etc..
         // TODO define attr name as constant somewhere.
         // TODO populate appropriate attributes.
         portEvent.createStringAttribute(TopologyElement.TYPE,
-                                        TopologyElement.TYPE_PACKET_LAYER);
+                TopologyElement.TYPE_PACKET_LAYER);
         portEvent.createStringAttribute("name", port.getName());
 
         portEvent.freeze();
 
         if (registryService.hasControl(switchId)) {
             topologyDiscoveryInterface.putPortDiscoveryEvent(portEvent);
-            linkDiscovery.enableDiscoveryOnPort(switchId, port.getPortNumber());
+            linkDiscovery.enableDiscoveryOnPort(switchId,
+                    port.getPortNo().getShortPortNumber());
         } else {
             log.debug("Not the master for switch {}. Suppressed port add event {}.",
                     new Dpid(switchId), portEvent);
         }
     }
 
-    @Override
-    public void switchPortRemoved(Long switchId, OFPhysicalPort port) {
+    private void switchPortRemoved(long switchId, OFPortDesc port) {
         final Dpid dpid = new Dpid(switchId);
 
-        PortEvent portEvent = new PortEvent(dpid, new PortNumber(port.getPortNumber()));
+        PortEvent portEvent = new PortEvent(dpid, new PortNumber(
+                port.getPortNo().getShortPortNumber()));
         // FIXME should be merging, with existing attrs, etc..
         // TODO define attr name as constant somewhere.
         // TODO populate appropriate attributes.
         portEvent.createStringAttribute(TopologyElement.TYPE,
-                                        TopologyElement.TYPE_PACKET_LAYER);
+                TopologyElement.TYPE_PACKET_LAYER);
         portEvent.createStringAttribute("name", port.getName());
 
         portEvent.freeze();
@@ -235,70 +356,8 @@
     }
 
     @Override
-    public void addedSwitch(IOFSwitch sw) {
-        final Dpid dpid = new Dpid(sw.getId());
-        SwitchEvent switchEvent = new SwitchEvent(dpid);
-        // FIXME should be merging, with existing attrs, etc..
-        // TODO define attr name as constant somewhere.
-        // TODO populate appropriate attributes.
-        switchEvent.createStringAttribute(TopologyElement.TYPE,
-                                          TopologyElement.TYPE_PACKET_LAYER);
-        switchEvent.createStringAttribute("ConnectedSince",
-                sw.getConnectedSince().toString());
-        switchEvent.createStringAttribute(TopologyElement.ELEMENT_CONFIG_STATE,
-                ConfigState.NOT_CONFIGURED.toString());
-        switchEvent.createStringAttribute(TopologyElement.ELEMENT_ADMIN_STATUS,
-                AdminStatus.ACTIVE.toString());
-        switchEvent.freeze();
-
-        // TODO Not very robust
-        if (!registryService.hasControl(sw.getId())) {
-            log.debug("Not the master for switch {}. Suppressed switch add event {}.",
-                    dpid, switchEvent);
-            return;
-        }
-
-        List<PortEvent> portEvents = new ArrayList<PortEvent>();
-        for (OFPhysicalPort port : sw.getPorts()) {
-            PortEvent portEvent = new PortEvent(dpid, new PortNumber(port.getPortNumber()));
-            // FIXME should be merging, with existing attrs, etc..
-            // TODO define attr name as constant somewhere.
-            // TODO populate appropriate attributes.
-            portEvent.createStringAttribute("name", port.getName());
-            portEvent.createStringAttribute(TopologyElement.TYPE,
-                                            TopologyElement.TYPE_PACKET_LAYER);
-            portEvent.createStringAttribute(TopologyElement.ELEMENT_CONFIG_STATE,
-                    ConfigState.NOT_CONFIGURED.toString());
-            portEvent.createStringAttribute(TopologyElement.ELEMENT_ADMIN_STATUS,
-                    AdminStatus.ACTIVE.toString());
-
-            portEvent.freeze();
-            portEvents.add(portEvent);
-        }
-        topologyDiscoveryInterface
-                .putSwitchDiscoveryEvent(switchEvent, portEvents);
-
-        for (OFPhysicalPort port : sw.getPorts()) {
-            // Allow links to be discovered on this port now that it's
-            // in the database
-            linkDiscovery.enableDiscoveryOnPort(sw.getId(), port.getPortNumber());
-        }
-    }
-
-    @Override
-    public void removedSwitch(IOFSwitch sw) {
-        // We don't use this event - switch remove is done by cleanup thread
-    }
-
-    @Override
-    public void switchPortChanged(Long switchId) {
-        // We don't use this event
-    }
-
-    @Override
     public String getName() {
-        // TODO Auto-generated method stub
-        return null;
+        return "topologyPublisher";
     }
 
     /* *****************
@@ -312,13 +371,13 @@
 
     @Override
     public Map<Class<? extends IFloodlightService>, IFloodlightService>
-    getServiceImpls() {
+            getServiceImpls() {
         return null;
     }
 
     @Override
     public Collection<Class<? extends IFloodlightService>>
-    getModuleDependencies() {
+            getModuleDependencies() {
         Collection<Class<? extends IFloodlightService>> l =
                 new ArrayList<Class<? extends IFloodlightService>>();
         l.add(IFloodlightProviderService.class);
@@ -339,8 +398,6 @@
         hostService = context.getServiceImpl(IHostService.class);
 
         topologyService = context.getServiceImpl(ITopologyService.class);
-
-        floodlightProvider.addLocalSwitchMastershipListener(this);
     }
 
     @Override
@@ -393,41 +450,23 @@
     public void hostRemoved(Host host) {
         log.debug("Called onosDeviceRemoved");
         HostEvent event = new HostEvent(host.getMacAddress());
-        //XXX shouldn't we be setting attachment points?
+        // XXX shouldn't we be setting attachment points?
         event.freeze();
         topologyDiscoveryInterface.removeHostDiscoveryEvent(event);
     }
 
-    @Override
-    public void controllerRoleChanged(Dpid dpid, Role role) {
-        log.debug("Local switch controller mastership role changed: dpid = {} role = {}", dpid, role);
+    private void controllerRoleChanged(Dpid dpid, Role role) {
+        log.debug("Local switch controller mastership role changed: dpid = {} role = {}",
+                dpid, role);
         MastershipEvent mastershipEvent =
-            new MastershipEvent(dpid, registryService.getOnosInstanceId(),
-                                role);
+                new MastershipEvent(dpid, registryService.getOnosInstanceId(),
+                        role);
         // FIXME should be merging, with existing attrs, etc..
         // TODO define attr name as constant somewhere.
         // TODO populate appropriate attributes.
         mastershipEvent.createStringAttribute(TopologyElement.TYPE,
-                                              TopologyElement.TYPE_ALL_LAYERS);
+                TopologyElement.TYPE_ALL_LAYERS);
         mastershipEvent.freeze();
         topologyDiscoveryInterface.putSwitchMastershipEvent(mastershipEvent);
     }
-
-    @Override
-    public void switchDisconnected(Dpid dpid) {
-        log.debug("Local switch disconnected: dpid = {} role = {}", dpid);
-
-        Role role = Role.SLAVE;         // TODO: Should be Role.UNKNOWN
-
-        MastershipEvent mastershipEvent =
-            new MastershipEvent(dpid, registryService.getOnosInstanceId(),
-                                role);
-        // FIXME should be merging, with existing attrs, etc..
-        // TODO define attr name as constant somewhere.
-        // TODO populate appropriate attributes.
-        mastershipEvent.createStringAttribute(TopologyElement.TYPE,
-                                              TopologyElement.TYPE_ALL_LAYERS);
-        mastershipEvent.freeze();
-        topologyDiscoveryInterface.removeSwitchMastershipEvent(mastershipEvent);
-    }
 }
diff --git a/src/main/resources/META-INF/services/net.floodlightcontroller.core.module.IFloodlightModule b/src/main/resources/META-INF/services/net.floodlightcontroller.core.module.IFloodlightModule
index b7636ab..3fccd77 100644
--- a/src/main/resources/META-INF/services/net.floodlightcontroller.core.module.IFloodlightModule
+++ b/src/main/resources/META-INF/services/net.floodlightcontroller.core.module.IFloodlightModule
@@ -3,6 +3,8 @@
 net.onrc.onos.core.linkdiscovery.LinkDiscoveryManager
 net.floodlightcontroller.restserver.RestApiServer
 net.floodlightcontroller.threadpool.ThreadPool
+net.floodlightcontroller.debugcounter.DebugCounter
+net.floodlightcontroller.debugevent.DebugEvent
 net.floodlightcontroller.core.test.MockFloodlightProvider
 net.floodlightcontroller.core.test.MockThreadPoolService
 net.onrc.onos.core.datagrid.HazelcastDatagrid