Renamed forwarding and proxyarp packages

net.onrc.onos.ofcontroller.forwarding => net.onrc.onos.apps.forwarding
net.onrc.onos.ofcontroller.proxyarp => net.onrc.onos.apps.proxyarp

Change-Id: Id368d4fd675b00ad84c17d44dd9804f010710cde
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/ArpCache.java b/src/main/java/net/onrc/onos/apps/proxyarp/ArpCache.java
new file mode 100644
index 0000000..f2e2891
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/ArpCache.java
@@ -0,0 +1,143 @@
+package net.onrc.onos.apps.proxyarp;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.floodlightcontroller.util.MACAddress;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * TODO clean out old ARP entries out of the cache periodically. We currently
+ * don't do this which means the cache memory size will never decrease. We
+ * already have a periodic thread that can be used to do this in
+ * ProxyArpManager.
+ */
+
+/**
+ * Implements a basic ARP cache which maps IPv4 addresses to MAC addresses.
+ * Mappings time out after a short period of time (currently 1 min). We don't
+ * try and refresh the mapping before the entry times out because as a
+ * controller we don't know if the mapping is still needed.
+ */
+class ArpCache {
+    private static final Logger log = LoggerFactory.getLogger(ArpCache.class);
+
+    private static final long ARP_ENTRY_TIMEOUT = 60000; // ms (1 min)
+
+    // Protected by locking on the ArpCache object (this)
+    private final Map<InetAddress, ArpCacheEntry> arpCache;
+
+    /**
+     * Represents a MAC address entry with a timestamp in the ARP cache.
+     * ARP cache entries are considered invalid if their timestamp is older
+     * than a timeout value.
+     */
+    private static class ArpCacheEntry {
+        private final MACAddress macAddress;
+        private long timeLastSeen;
+
+        /**
+         * Class constructor, specifying the MAC address for the entry.
+         * @param macAddress MAC address for the entry
+         */
+        public ArpCacheEntry(MACAddress macAddress) {
+            this.macAddress = macAddress;
+            this.timeLastSeen = System.currentTimeMillis();
+        }
+
+        /**
+         * Returns the MAC address this entry represents.
+         * @return this entry's MAC address
+         */
+        public MACAddress getMacAddress() {
+            return macAddress;
+        }
+
+        /**
+         * Update the timestamp for this entry.
+         * @param time the new timestamp to update the entry with
+         */
+        public void setTimeLastSeen(long time) {
+            timeLastSeen = time;
+        }
+
+        /**
+         * Returns whether the entry has timed out or not.
+         * @return true if the entry has timed out.
+         */
+        public boolean isExpired() {
+            return System.currentTimeMillis() - timeLastSeen > ARP_ENTRY_TIMEOUT;
+        }
+    }
+
+    /**
+     * Class constructor.
+     */
+    ArpCache() {
+        arpCache = new HashMap<InetAddress, ArpCacheEntry>();
+    }
+
+    /**
+     * Get the MAC address that is mapped to an IP address in the ARP cache.
+     * @param ipAddress the IP address to look up
+     * @return the MAC address if found in the cache, null if not
+     */
+    synchronized MACAddress lookup(InetAddress ipAddress) {
+        ArpCacheEntry arpEntry = arpCache.get(ipAddress);
+
+        if (arpEntry == null) {
+            return null;
+        }
+
+        if (arpEntry.isExpired()) {
+            // Entry has timed out so we'll remove it and return null
+            log.trace("Removing expired ARP entry for {}",
+                    ipAddress.getHostAddress());
+
+            arpCache.remove(ipAddress);
+            return null;
+        }
+
+        return arpEntry.getMacAddress();
+    }
+
+    /**
+     * Update an entry in the ARP cache. If the IP to MAC mapping is already
+     * in the cache, its timestamp will be updated. If not, the entry will
+     * be added with a new timestamp of the current time.
+     * @param ipAddress the IP address that will be mapped in the cache
+     * @param macAddress the MAC address that maps to {@code ipAddress}
+     */
+    synchronized void update(InetAddress ipAddress, MACAddress macAddress) {
+        ArpCacheEntry arpEntry = arpCache.get(ipAddress);
+
+        if (arpEntry != null && arpEntry.getMacAddress().equals(macAddress)) {
+            arpEntry.setTimeLastSeen(System.currentTimeMillis());
+        } else {
+            arpCache.put(ipAddress, new ArpCacheEntry(macAddress));
+        }
+    }
+
+    /**
+     * Retrieve a list of all mappings in the ARP cache.
+     * @return list of all ARP mappings, formatted as a human-readable string
+     *
+     */
+    synchronized List<String> getMappings() {
+        List<String> result = new ArrayList<String>(arpCache.size());
+
+        for (Map.Entry<InetAddress, ArpCacheEntry> entry : arpCache.entrySet()) {
+            result.add(entry.getKey().getHostAddress()
+                    + " => "
+                    + entry.getValue().getMacAddress().toString()
+                    + (entry.getValue().isExpired() ? " : EXPIRED" : " : VALID"));
+        }
+
+        return result;
+    }
+}
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/ArpCacheResource.java b/src/main/java/net/onrc/onos/apps/proxyarp/ArpCacheResource.java
new file mode 100644
index 0000000..7bf2a5a
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/ArpCacheResource.java
@@ -0,0 +1,26 @@
+package net.onrc.onos.apps.proxyarp;
+
+import java.util.List;
+
+import org.restlet.resource.Get;
+import org.restlet.resource.ServerResource;
+
+/**
+ * REST resource to view the IP to MAC mappings in the ARP cache.
+ *
+ */
+public class ArpCacheResource extends ServerResource {
+
+    /**
+     * Handler for a REST call to retrieve the ARP cache.
+     * @return list of mappings formatted as a human-readable string.
+     */
+    @Get("json")
+    public List<String> getArpCache() {
+        IProxyArpService arp = (IProxyArpService) getContext().getAttributes()
+                .get(IProxyArpService.class.getCanonicalName());
+
+        return arp.getMappings();
+    }
+
+}
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/ArpReplyNotification.java b/src/main/java/net/onrc/onos/apps/proxyarp/ArpReplyNotification.java
new file mode 100644
index 0000000..bde2734
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/ArpReplyNotification.java
@@ -0,0 +1,46 @@
+package net.onrc.onos.apps.proxyarp;
+
+import java.io.Serializable;
+
+import net.floodlightcontroller.util.MACAddress;
+
+/**
+ * Inter-instance notification that an ARP reply has been received. The
+ * notification contains both the IP address and the MAC address.
+ */
+public class ArpReplyNotification implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private int targetAddress;
+    private MACAddress targetMacAddress;
+
+    protected ArpReplyNotification() {}
+    /**
+     * Class constructor.
+     * @param targetAddress IP address received from the ARP reply
+     * @param targetMacAddress MAC address received from the ARP reply
+     */
+    public ArpReplyNotification(int targetAddress,
+            MACAddress targetMacAddress) {
+        this.targetAddress = targetAddress;
+        this.targetMacAddress = targetMacAddress;
+    }
+
+    /**
+     * Returns the IP address of the ARP reply.
+     * @return the IP address
+     */
+    public int getTargetAddress() {
+        return targetAddress;
+    }
+
+    /**
+     * Returns the MAC address of the ARP reply.
+     * @return the MAC address
+     */
+    public MACAddress getTargetMacAddress() {
+        return targetMacAddress;
+    }
+
+}
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/ArpWebRoutable.java b/src/main/java/net/onrc/onos/apps/proxyarp/ArpWebRoutable.java
new file mode 100644
index 0000000..82847f2
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/ArpWebRoutable.java
@@ -0,0 +1,36 @@
+package net.onrc.onos.apps.proxyarp;
+
+import net.floodlightcontroller.restserver.RestletRoutable;
+
+import org.restlet.Context;
+import org.restlet.Restlet;
+import org.restlet.routing.Router;
+
+/**
+ * Routing class for ARP module REST URLs.
+ */
+public class ArpWebRoutable implements RestletRoutable {
+
+    /**
+     * Get a router configured with ARP module REST URLs.
+     *
+     * @param context the restlet context to build a router with
+     * @return the router
+     */
+    @Override
+    public Restlet getRestlet(Context context) {
+        Router router = new Router(context);
+        router.attach("/cache/json", ArpCacheResource.class);
+        return router;
+    }
+
+    /**
+     * Get the base path of the ARP module URLs.
+     *
+     * @return the string base path
+     */
+    @Override
+    public String basePath() {
+        return "/wm/arp";
+    }
+}
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/BroadcastPacketOutNotification.java b/src/main/java/net/onrc/onos/apps/proxyarp/BroadcastPacketOutNotification.java
new file mode 100644
index 0000000..c2097f2
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/BroadcastPacketOutNotification.java
@@ -0,0 +1,77 @@
+package net.onrc.onos.apps.proxyarp;
+
+
+
+// TODO This class is too generic to be handled by ProxyArpService.
+// TODO The generic broadcast packet shouldn't contain an IP address which is
+// only for ARP packets.
+/**
+ * Notification to all ONOS instances to broadcast this packet out the edge of
+ * the network. The edge is defined as any port that doesn't have a link to
+ * another switch. The one exception is the port that the packet was received
+ * on.
+ *
+ */
+public class BroadcastPacketOutNotification extends PacketOutNotification {
+
+    private static final long serialVersionUID = 1L;
+
+    private final int address;
+    private final long inSwitch;
+    private final short inPort;
+
+    protected BroadcastPacketOutNotification() {
+    	super();
+        this.address = -1;
+        this.inSwitch = -1;
+        this.inPort = -1;
+    }
+    /**
+     * Class constructor.
+     *
+     * @param packet
+     *        packet data to send in the packet-out
+     * @param address
+     *        target IP address if the packet is an ARP packet
+     * @param inSwitch
+     *        dpid of the switch the packet was received on
+     * @param inPort
+     *        port number of the receiving port
+     */
+    public BroadcastPacketOutNotification(byte[] packet, int address,
+            long inSwitch, short inPort) {
+        super(packet);
+
+        this.address = address;
+        this.inSwitch = inSwitch;
+        this.inPort = inPort;
+    }
+    
+    /**
+     * Get the dpid of the switch the packet was received on.
+     *
+     * @return receiving switch dpid
+     */
+    public long getInSwitch() {
+        return inSwitch;
+    }
+
+    /**
+     * Get the port number of the port the packet was received on.
+     *
+     * @return receiving port number
+     */
+    public short getInPort() {
+        return inPort;
+    }
+
+    /**
+     * Get the target IP address if the packet is an ARP packet.
+     *
+     * @return the target IP address for ARP packets, or null if the packet is
+     *         not an ARP packet
+     */
+    public int getTargetAddress() {
+        return address;
+    }
+}
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/IArpRequester.java b/src/main/java/net/onrc/onos/apps/proxyarp/IArpRequester.java
new file mode 100644
index 0000000..ddc0ab4
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/IArpRequester.java
@@ -0,0 +1,24 @@
+package net.onrc.onos.apps.proxyarp;
+
+import java.net.InetAddress;
+
+import net.floodlightcontroller.util.MACAddress;
+
+/**
+ * Callback interface for modules using the {@link IProxyArpService} to send ARP
+ * requests.
+ *
+ */
+public interface IArpRequester {
+    /**
+     * Callback method that will be called by the {@link IProxyArpService} when
+     * it receives a reply for a request previously submitted by this
+     * {@code IArpRequester}.
+     *
+     * @param ipAddress
+     *        The IP address than an ARP request was sent for
+     * @param macAddress
+     *        The MAC address mapped to the requested IP address
+     */
+    public void arpResponse(InetAddress ipAddress, MACAddress macAddress);
+}
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/IProxyArpService.java b/src/main/java/net/onrc/onos/apps/proxyarp/IProxyArpService.java
new file mode 100644
index 0000000..1cdf18c
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/IProxyArpService.java
@@ -0,0 +1,42 @@
+package net.onrc.onos.apps.proxyarp;
+
+import java.net.InetAddress;
+import java.util.List;
+
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.util.MACAddress;
+
+// Extends IFloodlightService so we can access it from REST API resources
+/**
+ * Provides ARP services to other modules.
+ */
+public interface IProxyArpService extends IFloodlightService {
+    /**
+     * Returns the MAC address if there is a valid entry in the cache. Otherwise
+     * returns null.
+     *
+     * @param ipAddress the IP address to request the ARP mapping for
+     * @return the MACAddress that maps to the specified IP address, or null if
+     *         no mapping is found
+     */
+    public MACAddress getMacAddress(InetAddress ipAddress);
+
+    /**
+     * Tell the IProxyArpService to send an ARP request for the IP address. The
+     * request will be broadcast out all edge ports in the network.
+     *
+     * @param ipAddress the IP address to send an ARP request for
+     * @param requester the {@link IArpRequester} object that will be called if
+     *                  a reply is received
+     * @param retry whether to keep sending requests until the MAC is learnt
+     */
+    public void sendArpRequest(InetAddress ipAddress, IArpRequester requester,
+            boolean retry);
+
+    /**
+     * Returns a snapshot of the entire ARP cache.
+     *
+     * @return a list of mappings formatted as a human-readable string
+     */
+    public List<String> getMappings();
+}
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/PacketOutNotification.java b/src/main/java/net/onrc/onos/apps/proxyarp/PacketOutNotification.java
new file mode 100644
index 0000000..bd64e59
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/PacketOutNotification.java
@@ -0,0 +1,28 @@
+package net.onrc.onos.apps.proxyarp;
+
+import java.io.Serializable;
+
+/**
+ * A PacketOutNotification contains data sent between ONOS instances that
+ * directs other instances to send a packet out a set of ports. This is an
+ * abstract base class that will be subclassed by specific types of
+ * notifications.
+ */
+public abstract class PacketOutNotification implements Serializable{
+
+    private static final long serialVersionUID = 1L;
+
+    protected final byte[] packet;
+    
+    /**
+     * Class constructor.
+     * @param packet the packet data to send in the packet-out
+     */
+    public PacketOutNotification() {
+    	packet = null;
+    }
+    
+    public PacketOutNotification(byte[] packet) {
+        this.packet = packet;
+    }
+}
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/ProxyArpManager.java b/src/main/java/net/onrc/onos/apps/proxyarp/ProxyArpManager.java
new file mode 100644
index 0000000..ef3bb83
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/ProxyArpManager.java
@@ -0,0 +1,936 @@
+package net.onrc.onos.apps.proxyarp;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import net.floodlightcontroller.core.FloodlightContext;
+import net.floodlightcontroller.core.IFloodlightProviderService;
+import net.floodlightcontroller.core.IOFMessageListener;
+import net.floodlightcontroller.core.IOFSwitch;
+import net.floodlightcontroller.core.module.FloodlightModuleContext;
+import net.floodlightcontroller.core.module.IFloodlightModule;
+import net.floodlightcontroller.core.module.IFloodlightService;
+import net.floodlightcontroller.restserver.IRestApiService;
+import net.floodlightcontroller.util.MACAddress;
+import net.onrc.onos.apps.bgproute.Interface;
+import net.onrc.onos.datagrid.IDatagridService;
+import net.onrc.onos.datagrid.IEventChannel;
+import net.onrc.onos.datagrid.IEventChannelListener;
+import net.onrc.onos.ofcontroller.core.config.IConfigInfoService;
+import net.onrc.onos.ofcontroller.devicemanager.IOnosDeviceService;
+import net.onrc.onos.ofcontroller.flowprogrammer.IFlowPusherService;
+import net.onrc.onos.ofcontroller.networkgraph.Device;
+import net.onrc.onos.ofcontroller.networkgraph.INetworkGraphService;
+import net.onrc.onos.ofcontroller.networkgraph.NetworkGraph;
+import net.onrc.onos.ofcontroller.networkgraph.Switch;
+import net.onrc.onos.ofcontroller.util.Dpid;
+import net.onrc.onos.ofcontroller.util.Port;
+import net.onrc.onos.ofcontroller.util.SwitchPort;
+import net.onrc.onos.packet.ARP;
+import net.onrc.onos.packet.Ethernet;
+import net.onrc.onos.packet.IPv4;
+
+import org.openflow.protocol.OFMessage;
+import org.openflow.protocol.OFPacketIn;
+import org.openflow.protocol.OFPacketOut;
+import org.openflow.protocol.OFPort;
+import org.openflow.protocol.OFType;
+import org.openflow.protocol.action.OFAction;
+import org.openflow.protocol.action.OFActionOutput;
+import org.openflow.util.HexString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.SetMultimap;
+
+public class ProxyArpManager implements IProxyArpService, IOFMessageListener,
+					IFloodlightModule {
+    private static final Logger log = LoggerFactory
+            .getLogger(ProxyArpManager.class);
+
+    private static final long ARP_TIMER_PERIOD = 100; // ms
+
+    private static final int ARP_REQUEST_TIMEOUT = 2000; // ms
+
+    private IFloodlightProviderService floodlightProvider;
+    private IDatagridService datagrid;
+    private IEventChannel<Long, ArpReplyNotification> arpReplyEventChannel;
+    private IEventChannel<Long, BroadcastPacketOutNotification> broadcastPacketOutEventChannel;
+    private IEventChannel<Long, SinglePacketOutNotification> singlePacketOutEventChannel;
+    private static final String ARP_REPLY_CHANNEL_NAME = "onos.arp_reply";
+    private static final String BROADCAST_PACKET_OUT_CHANNEL_NAME = "onos.broadcast_packet_out";
+    private static final String SINGLE_PACKET_OUT_CHANNEL_NAME = "onos.single_packet_out";
+    private ArpReplyEventHandler arpReplyEventHandler = new ArpReplyEventHandler();
+    private BroadcastPacketOutEventHandler broadcastPacketOutEventHandler = new BroadcastPacketOutEventHandler();  
+    private SinglePacketOutEventHandler singlePacketOutEventHandler = new SinglePacketOutEventHandler();
+
+    private IConfigInfoService configService;
+    private IRestApiService restApi;
+    private IFlowPusherService flowPusher;
+    
+	private INetworkGraphService networkGraphService;
+	private NetworkGraph networkGraph;
+	private IOnosDeviceService onosDeviceService;
+
+    private short vlan;
+    private static final short NO_VLAN = 0;
+
+    private SetMultimap<InetAddress, ArpRequest> arpRequests;
+
+    private class BroadcastPacketOutEventHandler implements
+    IEventChannelListener<Long, BroadcastPacketOutNotification> {
+
+		@Override
+		public void entryAdded(BroadcastPacketOutNotification value) {
+			if(log.isTraceEnabled()) {
+				log.trace("entryAdded ip{}, sw {}, port {}, packet {}", value.getTargetAddress(), value.getInSwitch(), value.getInPort(), value.packet.length);
+			}
+			BroadcastPacketOutNotification notification = (BroadcastPacketOutNotification) value;
+			broadcastArpRequestOutMyEdge(notification.packet,
+						     notification.getInSwitch(),
+						     notification.getInPort());
+		
+			// set timestamp
+			ByteBuffer buffer = ByteBuffer.allocate(4);
+			buffer.putInt(notification.getTargetAddress());
+			InetAddress addr = null;
+			try {
+				addr = InetAddress.getByAddress(buffer.array());
+			} catch (UnknownHostException e) {
+				log.error("Exception:", e);
+			}
+			
+			if (addr != null) {
+			    for (ArpRequest request : arpRequests.get(addr)) {
+			    	request.setRequestTime();
+			    }
+			}			
+		}
+		
+		@Override
+		public void entryUpdated(BroadcastPacketOutNotification value) {
+			log.debug("entryUpdated");
+		    // TODO: For now, entryUpdated() is processed as entryAdded()
+		    entryAdded(value);
+		}
+		
+		@Override
+		public void entryRemoved(BroadcastPacketOutNotification value) {
+			log.debug("entryRemoved");
+		    // TODO: Not implemented. Revisit when this module is refactored
+		}
+    }
+    
+    private class SinglePacketOutEventHandler implements
+		IEventChannelListener<Long, SinglePacketOutNotification> {
+		@Override
+		public void entryAdded(SinglePacketOutNotification packetOutNotification) {
+			log.debug("entryAdded");
+			SinglePacketOutNotification notification =
+			    (SinglePacketOutNotification) packetOutNotification;
+			sendArpRequestOutPort(notification.packet,
+					      notification.getOutSwitch(),
+					      notification.getOutPort());
+	
+			// set timestamp
+			ByteBuffer buffer = ByteBuffer.allocate(4);
+			buffer.putInt(notification.getTargetAddress());
+			InetAddress addr = null;
+			try {
+				addr = InetAddress.getByAddress(buffer.array());
+			} catch (UnknownHostException e) {
+				log.error("Exception:", e);
+			}
+			
+			if (addr != null) {
+			    for (ArpRequest request : arpRequests.get(addr)) {
+			    	request.setRequestTime();
+			    }
+			}		
+		}
+	
+		@Override
+		public void entryUpdated(SinglePacketOutNotification packetOutNotification) {
+			log.debug("entryUpdated");
+		    // TODO: For now, entryUpdated() is processed as entryAdded()
+		    entryAdded(packetOutNotification);
+		}
+	
+		@Override
+		public void entryRemoved(SinglePacketOutNotification packetOutNotification) {
+			log.debug("entryRemoved");
+		    // TODO: Not implemented. Revisit when this module is refactored
+		}
+    }
+
+    private class ArpReplyEventHandler implements
+	IEventChannelListener<Long, ArpReplyNotification> {
+    	
+	@Override
+	public void entryAdded(ArpReplyNotification arpReply) {
+	    log.debug("Received ARP reply notification for ip {}, mac {}",
+	    		arpReply.getTargetAddress(), arpReply.getTargetMacAddress());
+		ByteBuffer buffer = ByteBuffer.allocate(4);
+		buffer.putInt(arpReply.getTargetAddress());
+		InetAddress addr = null;
+		try {
+			addr = InetAddress.getByAddress(buffer.array());
+		} catch (UnknownHostException e) {
+			log.error("Exception:", e);
+		}
+	   
+		if(addr != null) {
+			sendArpReplyToWaitingRequesters(addr,
+				    arpReply.getTargetMacAddress());
+		}
+	}
+
+	@Override
+	public void entryUpdated(ArpReplyNotification arpReply) {
+	    // TODO: For now, entryUpdated() is processed as entryAdded()
+	    entryAdded(arpReply);
+	}
+
+	@Override
+	public void entryRemoved(ArpReplyNotification arpReply) {
+	    // TODO: Not implemented. Revisit when this module is refactored
+	}
+    }
+
+    private static class ArpRequest {
+        private final IArpRequester requester;
+        private final boolean retry;
+        private boolean sent = false;
+        private long requestTime;
+
+        public ArpRequest(IArpRequester requester, boolean retry) {
+            this.requester = requester;
+            this.retry = retry;
+        }
+
+        public ArpRequest(ArpRequest old) {
+            this.requester = old.requester;
+            this.retry = old.retry;
+        }
+
+        public boolean isExpired() {
+            return sent
+                    && ((System.currentTimeMillis() - requestTime) > ARP_REQUEST_TIMEOUT);
+        }
+
+        public boolean shouldRetry() {
+            return retry;
+        }
+
+        public void dispatchReply(InetAddress ipAddress,
+                MACAddress replyMacAddress) {
+            requester.arpResponse(ipAddress, replyMacAddress);
+        }
+
+        public void setRequestTime() {
+            this.requestTime = System.currentTimeMillis();
+            this.sent = true;
+        }
+    }
+
+    private class HostArpRequester implements IArpRequester {
+        private final ARP arpRequest;
+        private final long dpid;
+        private final short port;
+
+        public HostArpRequester(ARP arpRequest, long dpid, short port) {
+            this.arpRequest = arpRequest;
+            this.dpid = dpid;
+            this.port = port;
+        }
+
+        @Override
+        public void arpResponse(InetAddress ipAddress, MACAddress macAddress) {
+            ProxyArpManager.this.sendArpReply(arpRequest, dpid, port,
+                    macAddress);
+        }
+        
+		public ARP getArpRequest() {
+			return arpRequest;
+		}
+    }
+
+    @Override
+    public Collection<Class<? extends IFloodlightService>> getModuleServices() {
+        Collection<Class<? extends IFloodlightService>> l =
+                new ArrayList<Class<? extends IFloodlightService>>();
+        l.add(IProxyArpService.class);
+        return l;
+    }
+
+    @Override
+    public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() {
+        Map<Class<? extends IFloodlightService>, IFloodlightService> m =
+                new HashMap<Class<? extends IFloodlightService>, IFloodlightService>();
+        m.put(IProxyArpService.class, this);
+        return m;
+    }
+
+    @Override
+    public Collection<Class<? extends IFloodlightService>> getModuleDependencies() {
+        Collection<Class<? extends IFloodlightService>> dependencies =
+                new ArrayList<Class<? extends IFloodlightService>>();
+        dependencies.add(IFloodlightProviderService.class);
+        dependencies.add(IRestApiService.class);
+        dependencies.add(IDatagridService.class);
+        dependencies.add(IConfigInfoService.class);
+        dependencies.add(IFlowPusherService.class);
+        dependencies.add(INetworkGraphService.class);
+        dependencies.add(IOnosDeviceService.class);
+        return dependencies;
+    }
+
+    @Override
+    public void init(FloodlightModuleContext context) {
+        this.floodlightProvider = context.getServiceImpl(IFloodlightProviderService.class); 
+        this.configService = context.getServiceImpl(IConfigInfoService.class);
+        this.restApi = context.getServiceImpl(IRestApiService.class);
+        this.datagrid = context.getServiceImpl(IDatagridService.class);
+        this.flowPusher = context.getServiceImpl(IFlowPusherService.class);
+        this.networkGraphService = context.getServiceImpl(INetworkGraphService.class);
+        this.onosDeviceService = context.getServiceImpl(IOnosDeviceService.class);
+
+        // arpCache = new ArpCache();
+
+        arpRequests = Multimaps.synchronizedSetMultimap(HashMultimap
+                .<InetAddress, ArpRequest>create());
+
+    }
+
+    @Override
+    public void startUp(FloodlightModuleContext context) {
+        this.vlan = configService.getVlan();
+        log.info("vlan set to {}", this.vlan);
+
+        restApi.addRestletRoutable(new ArpWebRoutable());
+        floodlightProvider.addOFMessageListener(OFType.PACKET_IN, this);
+		networkGraph = networkGraphService.getNetworkGraph();
+		
+	//
+	// Event notification setup: channels and event handlers
+	//	
+	broadcastPacketOutEventChannel = datagrid.addListener(BROADCAST_PACKET_OUT_CHANNEL_NAME,
+			     broadcastPacketOutEventHandler,
+			     Long.class,
+			     BroadcastPacketOutNotification.class);
+	
+	singlePacketOutEventChannel = datagrid.addListener(SINGLE_PACKET_OUT_CHANNEL_NAME,
+			     singlePacketOutEventHandler,
+			     Long.class,
+			     SinglePacketOutNotification.class);
+	
+	arpReplyEventChannel = datagrid.addListener(ARP_REPLY_CHANNEL_NAME,
+						    arpReplyEventHandler,
+						    Long.class,
+						    ArpReplyNotification.class);
+
+        Timer arpTimer = new Timer("arp-processing");
+        arpTimer.scheduleAtFixedRate(new TimerTask() {
+            @Override
+            public void run() {
+                doPeriodicArpProcessing();
+            }
+        }, 0, ARP_TIMER_PERIOD);
+    }
+
+    /*
+     * Function that runs periodically to manage the asynchronous request mechanism.
+     * It basically cleans up old ARP requests if we don't get a response for them.
+     * The caller can designate that a request should be retried indefinitely, and
+     * this task will handle that as well.
+     */
+    private void doPeriodicArpProcessing() {
+        SetMultimap<InetAddress, ArpRequest> retryList = HashMultimap
+                .<InetAddress, ArpRequest>create();
+
+        // Have to synchronize externally on the Multimap while using an
+        // iterator,
+        // even though it's a synchronizedMultimap
+        synchronized (arpRequests) {
+            Iterator<Map.Entry<InetAddress, ArpRequest>> it = arpRequests
+                    .entries().iterator();
+
+            while (it.hasNext()) {
+                Map.Entry<InetAddress, ArpRequest> entry = it.next();
+                ArpRequest request = entry.getValue();
+                if (request.isExpired()) {
+                    log.debug("Cleaning expired ARP request for {}", entry
+                            .getKey().getHostAddress());
+
+					// If the ARP request is expired and then delete the device
+					// TODO check whether this is OK from this thread
+					HostArpRequester requester = (HostArpRequester) request.requester;
+					ARP req = requester.getArpRequest();
+					Device targetDev = networkGraph.getDeviceByMac(MACAddress.valueOf(req.getTargetHardwareAddress()));
+					if(targetDev != null) {
+						onosDeviceService.deleteOnosDeviceByMac(MACAddress.valueOf(req.getTargetHardwareAddress()));
+						if (log.isDebugEnabled()) {
+							log.debug("RemoveDevice: {} due to no have not recieve the ARP reply", targetDev.getMacAddress());
+						}
+					}
+
+                    it.remove();
+
+                    if (request.shouldRetry()) {
+                        retryList.put(entry.getKey(), request);
+                    }
+                }
+            }
+        }
+
+        for (Map.Entry<InetAddress, Collection<ArpRequest>> entry : retryList
+                .asMap().entrySet()) {
+
+            InetAddress address = entry.getKey();
+
+            log.debug("Resending ARP request for {}", address.getHostAddress());
+
+            // Only ARP requests sent by the controller will have the retry flag
+            // set, so for now we can just send a new ARP request for that
+            // address.
+            sendArpRequestForAddress(address);
+
+            for (ArpRequest request : entry.getValue()) {
+                arpRequests.put(address, new ArpRequest(request));
+            }
+        }
+    }
+
+    @Override
+    public String getName() {
+        return "proxyarpmanager";
+    }
+
+    @Override
+    public boolean isCallbackOrderingPrereq(OFType type, String name) {
+        if (type == OFType.PACKET_IN) {
+            return "devicemanager".equals(name)
+                    || "onosdevicemanager".equals(name);
+        } else {
+            return false;
+        }
+    }
+
+    @Override
+    public boolean isCallbackOrderingPostreq(OFType type, String name) {
+        return type == OFType.PACKET_IN && "onosforwarding".equals(name);
+    }
+
+    @Override
+    public Command receive(IOFSwitch sw, OFMessage msg, FloodlightContext cntx) {
+
+        OFPacketIn pi = (OFPacketIn) msg;
+
+        Ethernet eth = IFloodlightProviderService.bcStore.get(cntx,
+                IFloodlightProviderService.CONTEXT_PI_PAYLOAD);
+
+        if (eth.getEtherType() == Ethernet.TYPE_ARP) {
+            ARP arp = (ARP) eth.getPayload();
+            if (arp.getOpCode() == ARP.OP_REQUEST) {
+                handleArpRequest(sw, pi, arp, eth);
+            } else if (arp.getOpCode() == ARP.OP_REPLY) {
+                // For replies we simply send a notification via Hazelcast
+                sendArpReplyNotification(eth, pi);
+
+                // handleArpReply(sw, pi, arp);
+            }
+
+            // Stop ARP packets here
+            return Command.STOP;
+        }
+
+        // Propagate everything else
+        return Command.CONTINUE;
+    }
+
+    private void handleArpRequest(IOFSwitch sw, OFPacketIn pi, ARP arp,
+            Ethernet eth) {
+        if (log.isTraceEnabled()) {
+            log.trace("ARP request received for {}",
+                    inetAddressToString(arp.getTargetProtocolAddress()));
+        }
+
+        InetAddress target;
+        try {
+            target = InetAddress.getByAddress(arp.getTargetProtocolAddress());
+        } catch (UnknownHostException e) {
+            log.debug("Invalid address in ARP request", e);
+            return;
+        }
+
+        if (configService.fromExternalNetwork(sw.getId(), pi.getInPort())) {
+            // If the request came from outside our network, we only care if
+            // it was a request for one of our interfaces.
+            if (configService.isInterfaceAddress(target)) {
+                log.trace(
+                        "ARP request for our interface. Sending reply {} => {}",
+                        target.getHostAddress(),
+                        configService.getRouterMacAddress());
+
+                sendArpReply(arp, sw.getId(), pi.getInPort(),
+                        configService.getRouterMacAddress());
+            }
+
+            return;
+        }
+
+        // MACAddress macAddress = arpCache.lookup(target);
+
+		arpRequests.put(target, new ArpRequest(
+				new HostArpRequester(arp, sw.getId(), pi.getInPort()), false));
+		
+		Device targetDevice = networkGraph.getDeviceByMac(MACAddress.valueOf(arp.getTargetHardwareAddress()));
+
+		if (targetDevice == null) {
+			if (log.isTraceEnabled()) {
+				log.trace("No device info found for {} - broadcasting",
+						target.getHostAddress());
+			}
+			
+			// We don't know the device so broadcast the request out
+			BroadcastPacketOutNotification key =
+					new BroadcastPacketOutNotification(eth.serialize(),
+							ByteBuffer.wrap(arp.getTargetProtocolAddress()).getInt(), sw.getId(), pi.getInPort());
+			log.debug("broadcastPacketOutEventChannel mac {}, ip {}, dpid {}, port {}, paket {}", eth.getSourceMAC().toLong(), 
+					ByteBuffer.wrap(arp.getTargetProtocolAddress()).getInt(), sw.getId(), pi.getInPort(), eth.serialize().length);
+			broadcastPacketOutEventChannel.addTransientEntry(eth.getDestinationMAC().toLong(), key);
+		}
+		else {
+			// Even if the device exists in our database, we do not reply to
+			// the request directly, but check whether the device is still valid
+			MACAddress macAddress = MACAddress.valueOf(arp.getTargetHardwareAddress());
+
+			if (log.isTraceEnabled()) {
+				log.trace("The target Device Record in DB is: {} => {} from ARP request host at {}/{}",
+						new Object [] {
+						inetAddressToString(arp.getTargetProtocolAddress()),
+						macAddress,
+						HexString.toHexString(sw.getId()), pi.getInPort()});
+			}
+
+			// sendArpReply(arp, sw.getId(), pi.getInPort(), macAddress);
+
+			Iterable<net.onrc.onos.ofcontroller.networkgraph.Port> outPorts = targetDevice.getAttachmentPoints();
+
+			if (!outPorts.iterator().hasNext()){
+				if (log.isTraceEnabled()) {
+					log.trace("Device {} exists but is not connected to any ports" + 
+							" - broadcasting", macAddress);
+				}
+				
+//				BroadcastPacketOutNotification key =
+//						new BroadcastPacketOutNotification(eth.serialize(), 
+//								target, sw.getId(), pi.getInPort());
+//				broadcastPacketOutEventChannel.addTransientEntry(eth.getDestinationMAC().toLong(), key);
+			} 
+			else {
+				for (net.onrc.onos.ofcontroller.networkgraph.Port portObject : outPorts) {
+					//long outSwitch = 0;
+					//short outPort = 0;
+
+					if(portObject.getOutgoingLink() != null || portObject.getIncomingLink() != null) {
+						continue;
+					}
+					
+					short outPort = portObject.getNumber().shortValue();
+					Switch outSwitchObject = portObject.getSwitch();
+					long outSwitch = outSwitchObject.getDpid();
+					
+					if (log.isTraceEnabled()) {
+						log.trace("Probing device {} on port {}/{}", 
+								new Object[] {macAddress, 
+								HexString.toHexString(outSwitch), outPort});
+					}
+					
+					SinglePacketOutNotification key =
+						    new SinglePacketOutNotification(eth.serialize(), 
+						    		ByteBuffer.wrap(target.getAddress()).getInt(), outSwitch, outPort);
+					singlePacketOutEventChannel.addTransientEntry(eth.getDestinationMAC().toLong(), key);
+				}
+			}
+		}
+    }
+
+    // Not used because device manager currently updates the database
+    // for ARP replies. May be useful in the future.
+    private void handleArpReply(IOFSwitch sw, OFPacketIn pi, ARP arp) {
+        if (log.isTraceEnabled()) {
+            log.trace("ARP reply recieved: {} => {}, on {}/{}", new Object[] {
+                    inetAddressToString(arp.getSenderProtocolAddress()),
+                    HexString.toHexString(arp.getSenderHardwareAddress()),
+                    HexString.toHexString(sw.getId()), pi.getInPort()});
+        }
+
+        InetAddress senderIpAddress;
+        try {
+            senderIpAddress = InetAddress.getByAddress(arp
+                    .getSenderProtocolAddress());
+        } catch (UnknownHostException e) {
+            log.debug("Invalid address in ARP reply", e);
+            return;
+        }
+
+        MACAddress senderMacAddress = MACAddress.valueOf(arp
+                .getSenderHardwareAddress());
+
+        // See if anyone's waiting for this ARP reply
+        Set<ArpRequest> requests = arpRequests.get(senderIpAddress);
+
+        // Synchronize on the Multimap while using an iterator for one of the
+        // sets
+        List<ArpRequest> requestsToSend = new ArrayList<ArpRequest>(
+                requests.size());
+        synchronized (arpRequests) {
+            Iterator<ArpRequest> it = requests.iterator();
+            while (it.hasNext()) {
+                ArpRequest request = it.next();
+                it.remove();
+                requestsToSend.add(request);
+            }
+        }
+
+        // Don't hold an ARP lock while dispatching requests
+        for (ArpRequest request : requestsToSend) {
+            request.dispatchReply(senderIpAddress, senderMacAddress);
+        }
+    }
+
+    private void sendArpRequestForAddress(InetAddress ipAddress) {
+        // TODO what should the sender IP address and MAC address be if no
+        // IP addresses are configured? Will there ever be a need to send
+        // ARP requests from the controller in that case?
+        // All-zero MAC address doesn't seem to work - hosts don't respond to it
+
+        byte[] zeroIpv4 = {0x0, 0x0, 0x0, 0x0};
+        byte[] zeroMac = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
+        byte[] genericNonZeroMac = {0x0, 0x0, 0x0, 0x0, 0x0, 0x01};
+        byte[] broadcastMac = {(byte) 0xff, (byte) 0xff, (byte) 0xff,
+                (byte) 0xff, (byte) 0xff, (byte) 0xff};
+
+        ARP arpRequest = new ARP();
+
+        arpRequest
+                .setHardwareType(ARP.HW_TYPE_ETHERNET)
+                .setProtocolType(ARP.PROTO_TYPE_IP)
+                .setHardwareAddressLength(
+                        (byte) Ethernet.DATALAYER_ADDRESS_LENGTH)
+                .setProtocolAddressLength((byte) IPv4.ADDRESS_LENGTH)
+                .setOpCode(ARP.OP_REQUEST).setTargetHardwareAddress(zeroMac)
+                .setTargetProtocolAddress(ipAddress.getAddress());
+
+        MACAddress routerMacAddress = configService.getRouterMacAddress();
+        // TODO hack for now as it's unclear what the MAC address should be
+        byte[] senderMacAddress = genericNonZeroMac;
+        if (routerMacAddress != null) {
+            senderMacAddress = routerMacAddress.toBytes();
+        }
+        arpRequest.setSenderHardwareAddress(senderMacAddress);
+
+        byte[] senderIPAddress = zeroIpv4;
+        Interface intf = configService.getOutgoingInterface(ipAddress);
+        if (intf != null) {
+            senderIPAddress = intf.getIpAddress().getAddress();
+        }
+
+        arpRequest.setSenderProtocolAddress(senderIPAddress);
+
+        Ethernet eth = new Ethernet();
+        eth.setSourceMACAddress(senderMacAddress)
+                .setDestinationMACAddress(broadcastMac)
+                .setEtherType(Ethernet.TYPE_ARP).setPayload(arpRequest);
+
+        if (vlan != NO_VLAN) {
+            eth.setVlanID(vlan).setPriorityCode((byte) 0);
+        }
+
+        // sendArpRequestToSwitches(ipAddress, eth.serialize());
+		SinglePacketOutNotification key =
+		    new SinglePacketOutNotification(eth.serialize(), ByteBuffer.wrap(ipAddress.getAddress()).getInt(),
+						    intf.getDpid(), intf.getPort());
+		singlePacketOutEventChannel.addTransientEntry(MACAddress.valueOf(senderMacAddress).toLong(), key);
+    }
+    
+    private void sendArpRequestToSwitches(InetAddress dstAddress, byte[] arpRequest) {
+    		sendArpRequestToSwitches(dstAddress, arpRequest, 0,
+    		OFPort.OFPP_NONE.getValue());
+    }
+
+    private void sendArpRequestToSwitches(InetAddress dstAddress,
+            byte[] arpRequest, long inSwitch, short inPort) {
+
+        if (configService.hasLayer3Configuration()) {
+            Interface intf = configService.getOutgoingInterface(dstAddress);
+            if (intf == null) {
+                // TODO here it should be broadcast out all non-interface edge
+                // ports.
+                // I think we can assume that if it's not a request for an
+                // external
+                // network, it's an ARP for a host in our own network. So we
+                // want to
+                // send it out all edge ports that don't have an interface
+                // configured
+                // to ensure it reaches all hosts in our network.
+                log.debug("No interface found to send ARP request for {}",
+                        dstAddress.getHostAddress());
+            } else {
+                sendArpRequestOutPort(arpRequest, intf.getDpid(),
+                        intf.getPort());
+            }
+        } else {
+            // broadcastArpRequestOutEdge(arpRequest, inSwitch, inPort);
+            broadcastArpRequestOutMyEdge(arpRequest, inSwitch, inPort);
+        }
+    }
+
+    private void sendArpReplyNotification(Ethernet eth, OFPacketIn pi) {
+        ARP arp = (ARP) eth.getPayload();
+
+        if (log.isTraceEnabled()) {
+            log.trace("Sending ARP reply for {} to other ONOS instances",
+                    inetAddressToString(arp.getSenderProtocolAddress()));
+        }
+
+        InetAddress targetAddress;
+
+        try {
+            targetAddress = InetAddress.getByAddress(arp
+                    .getSenderProtocolAddress());
+        } catch (UnknownHostException e) {
+            log.error("Unknown host", e);
+            return;
+        }
+
+        MACAddress mac = new MACAddress(arp.getSenderHardwareAddress());
+
+		ArpReplyNotification key =
+		    new ArpReplyNotification(ByteBuffer.wrap(targetAddress.getAddress()).getInt(), mac);
+		log.debug("ArpReplyNotification ip {}, mac{}", ByteBuffer.wrap(targetAddress.getAddress()).getInt(), mac);
+		arpReplyEventChannel.addTransientEntry(mac.toLong(), key);
+    }
+
+    private void broadcastArpRequestOutMyEdge(byte[] arpRequest, long inSwitch,
+            short inPort) {
+        List<SwitchPort> switchPorts = new ArrayList<SwitchPort>();
+
+        for (IOFSwitch sw : floodlightProvider.getSwitches().values()) {
+
+            OFPacketOut po = new OFPacketOut();
+            po.setInPort(OFPort.OFPP_NONE).setBufferId(-1)
+                    .setPacketData(arpRequest);
+
+            List<OFAction> actions = new ArrayList<OFAction>();
+
+			Switch graphSw = networkGraph.getSwitch(sw.getId());
+			Collection<net.onrc.onos.ofcontroller.networkgraph.Port> ports = graphSw.getPorts();
+			
+			if (ports == null) {
+				continue;
+			}
+			
+			for (net.onrc.onos.ofcontroller.networkgraph.Port portObject : ports) {
+				if (portObject.getOutgoingLink() == null && portObject.getNumber() > 0) {
+					Long portNumber = portObject.getNumber();
+					
+					if (sw.getId() == inSwitch && portNumber.shortValue() == inPort) {
+						// This is the port that the ARP message came in,
+						// so don't broadcast out this port
+						continue;
+					}		
+					switchPorts.add(new SwitchPort(new Dpid(sw.getId()), 
+							new net.onrc.onos.ofcontroller.util.Port(portNumber.shortValue())));
+					actions.add(new OFActionOutput(portNumber.shortValue()));
+				}
+			}
+
+            po.setActions(actions);
+            short actionsLength = (short) (actions.size() * OFActionOutput.MINIMUM_LENGTH);
+            po.setActionsLength(actionsLength);
+            po.setLengthU(OFPacketOut.MINIMUM_LENGTH + actionsLength
+                    + arpRequest.length);
+
+            flowPusher.add(sw, po);
+        }
+
+        if (log.isTraceEnabled()) {
+            log.trace("Broadcast ARP request to: {}", switchPorts);
+        }
+    }
+
+    private void sendArpRequestOutPort(byte[] arpRequest, long dpid, short port) {
+        if (log.isTraceEnabled()) {
+            log.trace("Sending ARP request out {}/{}",
+                    HexString.toHexString(dpid), port);
+        }
+
+        OFPacketOut po = new OFPacketOut();
+        po.setInPort(OFPort.OFPP_NONE).setBufferId(-1)
+                .setPacketData(arpRequest);
+
+        List<OFAction> actions = new ArrayList<OFAction>();
+        actions.add(new OFActionOutput(port));
+        po.setActions(actions);
+        short actionsLength = (short) (actions.size() * OFActionOutput.MINIMUM_LENGTH);
+        po.setActionsLength(actionsLength);
+        po.setLengthU(OFPacketOut.MINIMUM_LENGTH + actionsLength
+                + arpRequest.length);
+
+        IOFSwitch sw = floodlightProvider.getSwitches().get(dpid);
+
+        if (sw == null) {
+            log.warn("Switch not found when sending ARP request");
+            return;
+        }
+
+        flowPusher.add(sw, po);
+    }
+
+    private void sendArpReply(ARP arpRequest, long dpid, short port,
+            MACAddress targetMac) {
+        if (log.isTraceEnabled()) {
+            log.trace(
+                    "Sending reply {} => {} to {}",
+                    new Object[] {
+                            inetAddressToString(arpRequest
+                                    .getTargetProtocolAddress()),
+                            targetMac,
+                            inetAddressToString(arpRequest
+                                    .getSenderProtocolAddress())});
+        }
+
+        ARP arpReply = new ARP();
+        arpReply.setHardwareType(ARP.HW_TYPE_ETHERNET)
+                .setProtocolType(ARP.PROTO_TYPE_IP)
+                .setHardwareAddressLength(
+                        (byte) Ethernet.DATALAYER_ADDRESS_LENGTH)
+                .setProtocolAddressLength((byte) IPv4.ADDRESS_LENGTH)
+                .setOpCode(ARP.OP_REPLY)
+                .setSenderHardwareAddress(targetMac.toBytes())
+                .setSenderProtocolAddress(arpRequest.getTargetProtocolAddress())
+                .setTargetHardwareAddress(arpRequest.getSenderHardwareAddress())
+                .setTargetProtocolAddress(arpRequest.getSenderProtocolAddress());
+
+        Ethernet eth = new Ethernet();
+        eth.setDestinationMACAddress(arpRequest.getSenderHardwareAddress())
+                .setSourceMACAddress(targetMac.toBytes())
+                .setEtherType(Ethernet.TYPE_ARP).setPayload(arpReply);
+
+        if (vlan != NO_VLAN) {
+            eth.setVlanID(vlan).setPriorityCode((byte) 0);
+        }
+
+        List<OFAction> actions = new ArrayList<OFAction>();
+        actions.add(new OFActionOutput(port));
+
+        OFPacketOut po = new OFPacketOut();
+        po.setInPort(OFPort.OFPP_NONE)
+                .setBufferId(-1)
+                .setPacketData(eth.serialize())
+                .setActions(actions)
+                .setActionsLength((short) OFActionOutput.MINIMUM_LENGTH)
+                .setLengthU(
+                        OFPacketOut.MINIMUM_LENGTH
+                                + OFActionOutput.MINIMUM_LENGTH
+                                + po.getPacketData().length);
+
+        List<OFMessage> msgList = new ArrayList<OFMessage>();
+        msgList.add(po);
+
+        IOFSwitch sw = floodlightProvider.getSwitches().get(dpid);
+
+        if (sw == null) {
+            log.warn("Switch {} not found when sending ARP reply",
+                    HexString.toHexString(dpid));
+            return;
+        }
+
+        flowPusher.add(sw, po);
+    }
+
+    private String inetAddressToString(byte[] bytes) {
+        try {
+            return InetAddress.getByAddress(bytes).getHostAddress();
+        } catch (UnknownHostException e) {
+            log.debug("Invalid IP address", e);
+            return "";
+        }
+    }
+
+    /*
+     * IProxyArpService methods
+     */
+
+    @Override
+    public MACAddress getMacAddress(InetAddress ipAddress) {
+        // return arpCache.lookup(ipAddress);
+        return null;
+    }
+
+    @Override
+    public void sendArpRequest(InetAddress ipAddress, IArpRequester requester,
+            boolean retry) {
+        arpRequests.put(ipAddress, new ArpRequest(requester, retry));
+
+        // Sanity check to make sure we don't send a request for our own address
+        if (!configService.isInterfaceAddress(ipAddress)) {
+            sendArpRequestForAddress(ipAddress);
+        }
+    }
+
+    @Override
+    public List<String> getMappings() {
+        return new ArrayList<String>();
+    }
+
+    private void sendArpReplyToWaitingRequesters(InetAddress address,
+            MACAddress mac) {
+        log.debug("Sending ARP reply for {} to requesters",
+                address.getHostAddress());
+
+        // See if anyone's waiting for this ARP reply
+        Set<ArpRequest> requests = arpRequests.get(address);
+
+        // Synchronize on the Multimap while using an iterator for one of the
+        // sets
+        List<ArpRequest> requestsToSend = new ArrayList<ArpRequest>(
+                requests.size());
+        synchronized (arpRequests) {
+            Iterator<ArpRequest> it = requests.iterator();
+            while (it.hasNext()) {
+                ArpRequest request = it.next();
+                it.remove();
+                requestsToSend.add(request);
+            }
+        }
+
+        //TODO here, comment outed from long time ago. I will check if we need it later.
+        /*IDeviceObject deviceObject = deviceStorage.getDeviceByIP(
+        		InetAddresses.coerceToInteger(address));
+
+        MACAddress mac = MACAddress.valueOf(deviceObject.getMACAddress());
+
+        log.debug("Found {} at {} in network map",
+        		address.getHostAddress(), mac);*/
+
+        // Don't hold an ARP lock while dispatching requests
+        for (ArpRequest request : requestsToSend) {
+            request.dispatchReply(address, mac);
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/net/onrc/onos/apps/proxyarp/SinglePacketOutNotification.java b/src/main/java/net/onrc/onos/apps/proxyarp/SinglePacketOutNotification.java
new file mode 100644
index 0000000..8ee255b
--- /dev/null
+++ b/src/main/java/net/onrc/onos/apps/proxyarp/SinglePacketOutNotification.java
@@ -0,0 +1,57 @@
+package net.onrc.onos.apps.proxyarp;
+
+
+
+// TODO This class is too generic to be handled by ProxyArpService.
+/**
+ * Notification to another ONOS instance to send a packet out a single port.
+ */
+public class SinglePacketOutNotification extends PacketOutNotification {
+
+    private static final long serialVersionUID = 1L;
+
+    private final int address;
+    private final long outSwitch;
+    private final short outPort;
+
+    /**
+     * Class constructor.
+     * @param packet the packet data to send in the packet-out
+     * @param address target IP address if the packet is an ARP packet
+     * @param outSwitch the dpid of the switch to send the packet on
+     * @param outPort the port number of the port to send the packet out
+     */
+    public SinglePacketOutNotification(byte[] packet, int address,
+            long outSwitch, short outPort) {
+        super(packet);
+
+        this.address = address;
+        this.outSwitch = outSwitch;
+        this.outPort = outPort;
+    }
+
+    /**
+     * Get the dpid of the switch the packet will be sent out.
+     * @return the switch's dpid
+     */
+    public long getOutSwitch() {
+        return outSwitch;
+    }
+
+    /**
+     * Get the port number of the port the packet will be sent out.
+     * @return the port number
+     */
+    public short getOutPort() {
+        return outPort;
+    }
+
+    /**
+     * Get the target IP address if the packet is an ARP packet.
+     * @return the target IP address for ARP packets, or null if the packet is
+     *         not an ARP packet
+     */
+    public int getTargetAddress() {
+        return address;
+    }
+}