Merge pull request #539 from masayoshik/rest-fix

clean up rest server and related demo scripts
diff --git a/kryo2/.gitignore b/kryo2/.gitignore
deleted file mode 100644
index 916e17c..0000000
--- a/kryo2/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-dependency-reduced-pom.xml
diff --git a/kryo2/pom.xml b/kryo2/pom.xml
deleted file mode 100644
index d81a16a..0000000
--- a/kryo2/pom.xml
+++ /dev/null
@@ -1,69 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-
-  <groupId>net.onrc.onos</groupId>
-  <artifactId>kryo2</artifactId>
-  <version>2.22</version>
-  <packaging>jar</packaging>
-
-  <name>kryo2</name>
-  <url>http://maven.apache.org</url>
-
-  <properties>
-    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>com.esotericsoftware.kryo</groupId>
-      <artifactId>kryo</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-  </dependencies>
-  <build>
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <version>2.1</version>
-        <configuration>
-          <relocations>
-            <relocation>
-              <pattern>com.esotericsoftware.kryo</pattern>
-              <shadedPattern>com.esotericsoftware.kryo2</shadedPattern>
-              <excludes>
-                <exclude>com.esotericsoftware.kryo2*</exclude>
-              </excludes>
-            </relocation>
-          </relocations>
-        </configuration>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>exec-maven-plugin</artifactId>
-        <version>1.2.1</version>
-        <executions>
-          <execution>
-            <id>kryo2</id>
-            <goals>
-              <goal>exec</goal>
-            </goals>
-          </execution>
-        </executions>
-            <configuration>
-              <executable>mvn</executable>
-              <commandlineArgs>install:install-file -DlocalRepositoryPath=${basedir}/../repo -DcreateChecksum=true -Dpackaging=jar -Dfile=${basedir}/target/${project.build.finalName}.jar -DgroupId=${project.groupId} -DartifactId=${project.artifactId} -Dversion=${project.version}</commandlineArgs>
-            </configuration>
-      </plugin>
-    </plugins>
-  </build>
-</project>
diff --git a/pom.xml b/pom.xml
index c0cf489..6a1b29d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -265,22 +265,11 @@
     </plugins>
   </reporting>
   <dependencies>
-    <!-- Commenting out original kryo 2.X
-         and using shaded version (net.onrc.onos.kryo2)
-         to workaround conflict with kryo 1.X in titan's dependency.(#443)
-    -->
-    <!--
     <dependency>
       <groupId>com.esotericsoftware.kryo</groupId>
       <artifactId>kryo</artifactId>
       <version>2.22</version>
     </dependency>
-    -->
-    <dependency>
-      <groupId>net.onrc.onos</groupId>
-      <artifactId>kryo2</artifactId>
-      <version>2.22</version>
-    </dependency>
     <!-- ONOS's direct dependencies -->
     <dependency>
       <groupId>org.apache.cassandra</groupId>
diff --git a/rebuild-local-repo.sh b/rebuild-local-repo.sh
index 558bcfb..1baa27b 100755
--- a/rebuild-local-repo.sh
+++ b/rebuild-local-repo.sh
@@ -15,11 +15,6 @@
     MVN="mvn"
 fi
 
-# Install Kryo2 workaround to local repo
-# - Shaded(rename package name to allow mixing 2 different Kryo version)
-# - Install created sharded jar to local repo
-${MVN} -f kryo2/pom.xml clean package exec:exec
-
 # Install modified curators to local repo
 ${MVN} install:install-file -Dfile=./curator/curator-framework-1.3.5-SNAPSHOT.jar -DgroupId=com.netflix.curator -DartifactId=curator-framework -Dversion=1.3.5-SNAPSHOT -Dpackaging=jar -DgeneratePom=true -DlocalRepositoryPath=./repo -DcreateChecksum=true
 ${MVN} install:install-file -Dfile=./curator/curator-client-1.3.5-SNAPSHOT.jar -DgroupId=com.netflix.curator -DartifactId=curator-client -Dversion=1.3.5-SNAPSHOT -Dpackaging=jar -DgeneratePom=true -DlocalRepositoryPath=./repo -DcreateChecksum=true
diff --git a/src/main/java/net/onrc/onos/datagrid/HazelcastDatagrid.java b/src/main/java/net/onrc/onos/datagrid/HazelcastDatagrid.java
index 0c7be88..5be4191 100644
--- a/src/main/java/net/onrc/onos/datagrid/HazelcastDatagrid.java
+++ b/src/main/java/net/onrc/onos/datagrid/HazelcastDatagrid.java
@@ -32,9 +32,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.esotericsoftware.kryo2.Kryo;
-import com.esotericsoftware.kryo2.io.Input;
-import com.esotericsoftware.kryo2.io.Output;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
 import com.hazelcast.config.Config;
 import com.hazelcast.config.FileSystemXmlConfig;
 import com.hazelcast.core.EntryEvent;
@@ -80,14 +80,14 @@
     private IMap<String, byte[]> mapTopology = null;
     private MapTopologyListener mapTopologyListener = null;
     private String mapTopologyListenerId = null;
-    
+
     // State related to the packet out map
     protected static final String packetOutMapName = "packetOutMap";
     private IMap<PacketOutNotification, byte[]> packetOutMap = null;
     private List<IPacketOutEventHandler> packetOutEventHandlers = new ArrayList<IPacketOutEventHandler>();
 
     private final byte[] dummyByte = {0};
-    
+
     // State related to the ARP reply map
     protected static final String arpReplyMapName = "arpReplyMap";
     private IMap<ArpReplyNotification, byte[]> arpReplyMap = null;
@@ -337,6 +337,7 @@
 		 *
 		 * @param event the notification event for the entry.
 		 */
+		@Override
 		public void entryAdded(EntryEvent<PacketOutNotification, byte[]> event) {
 		    for (IPacketOutEventHandler packetOutEventHandler : packetOutEventHandlers) {
 		    	packetOutEventHandler.packetOutNotification(event.getKey());
@@ -348,6 +349,7 @@
 		 *
 		 * @param event the notification event for the entry.
 		 */
+		@Override
 		public void entryRemoved(EntryEvent<PacketOutNotification, byte[]> event) {
 			// Not used
 		}
@@ -357,6 +359,7 @@
 		 *
 		 * @param event the notification event for the entry.
 		 */
+		@Override
 		public void entryUpdated(EntryEvent<PacketOutNotification, byte[]> event) {
 			// Not used
 		}
@@ -366,11 +369,12 @@
 		 *
 		 * @param event the notification event for the entry.
 		 */
+		@Override
 		public void entryEvicted(EntryEvent<PacketOutNotification, byte[]> event) {
 		    // Not used
 		}
     }
-    
+
     /**
      * Class for receiving notifications for sending packet-outs.
      *
@@ -384,15 +388,19 @@
 		 *
 		 * @param event the notification event for the entry.
 		 */
+		@Override
 		public void entryAdded(EntryEvent<ArpReplyNotification, byte[]> event) {
 		    for (IArpReplyEventHandler arpReplyEventHandler : arpReplyEventHandlers) {
 		    	arpReplyEventHandler.arpReplyEvent(event.getKey());
 		    }
 		}
-		
+
 		// These methods aren't used for ARP replies
+		@Override
 		public void entryRemoved(EntryEvent<ArpReplyNotification, byte[]> event) {}
+		@Override
 		public void entryUpdated(EntryEvent<ArpReplyNotification, byte[]> event) {}
+		@Override
 		public void entryEvicted(EntryEvent<ArpReplyNotification, byte[]> event) {}
     }
 
@@ -511,10 +519,10 @@
 	hazelcastInstance = Hazelcast.newHazelcastInstance(hazelcastConfig);
 
 	restApi.addRestletRoutable(new DatagridWebRoutable());
-	
+
 	packetOutMap = hazelcastInstance.getMap(packetOutMapName);
 	packetOutMap.addEntryListener(new PacketOutMapListener(), true);
-	
+
 	arpReplyMap = hazelcastInstance.getMap(arpReplyMapName);
 	arpReplyMap.addEntryListener(new ArpReplyMapListener(), true);
     }
@@ -582,12 +590,12 @@
     		packetOutEventHandlers.add(packetOutEventHandler);
     	}
     }
-    
+
     @Override
     public void deregisterPacketOutEventHandler(IPacketOutEventHandler packetOutEventHandler) {
     	packetOutEventHandlers.remove(packetOutEventHandler);
     }
-    
+
     @Override
     public void registerArpReplyEventHandler(IArpReplyEventHandler arpReplyEventHandler) {
     	if (arpReplyEventHandler != null) {
diff --git a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowEventHandler.java b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowEventHandler.java
index 8b1f7c0..7550cfd 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowEventHandler.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowEventHandler.java
@@ -31,7 +31,7 @@
 import net.onrc.onos.ofcontroller.util.FlowPathUserState;
 import net.onrc.onos.ofcontroller.util.serializers.KryoFactory;
 
-import com.esotericsoftware.kryo2.Kryo;
+import com.esotericsoftware.kryo.Kryo;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -273,6 +273,10 @@
 
 	for (FlowPath flowPath : flowPaths) {
 	    boolean isInstalled = true;
+	    
+	    if (flowPath.flowEntries().isEmpty()) {
+	    	continue;
+	    }
 
 	    //
 	    // Check whether all Flow Entries have been installed
diff --git a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
index dd98f4e..f266163 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
@@ -39,7 +39,7 @@
 
 import com.thinkaurelius.titan.core.TitanException;
 
-import com.esotericsoftware.kryo2.Kryo;
+import com.esotericsoftware.kryo.Kryo;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -400,6 +400,11 @@
 	if (srcDpid.value() != sw.getId())
 	    return;
 	deleteFlow(flowPath.flowId());
+	
+	// Send flow deleted notification to the Forwarding module
+	// TODO This is a quick fix for flow-removed notifications. We
+	// should think more about the design of these notifications.
+	notificationFlowPathRemoved(flowPath);
     }
 
     /**
@@ -469,6 +474,20 @@
     }
 
     /**
+     * Generate a notification that a FlowPath has been removed from the 
+     * network. This means we've received an expiry message for the flow
+     * from the switch, and send flowmods to remove any remaining parts of
+     * the path.
+     * 
+     * @param flowPath FlowPath object that was removed from the network.
+     */
+    void notificationFlowPathRemoved(FlowPath flowPath) {
+	if (forwardingService != null) {
+		forwardingService.flowRemoved(flowPath);
+	}
+    }
+
+    /**
      * Push modified Flow-related state as appropriate.
      *
      * @param modifiedFlowPaths the collection of modified Flow Paths.
diff --git a/src/main/java/net/onrc/onos/ofcontroller/forwarding/Forwarding.java b/src/main/java/net/onrc/onos/ofcontroller/forwarding/Forwarding.java
index 6d953f1..b3d9759 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/forwarding/Forwarding.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/forwarding/Forwarding.java
@@ -25,12 +25,13 @@
 import net.onrc.onos.ofcontroller.devicemanager.IOnosDeviceService;
 import net.onrc.onos.ofcontroller.flowmanager.IFlowService;
 import net.onrc.onos.ofcontroller.flowprogrammer.IFlowPusherService;
-import net.onrc.onos.ofcontroller.proxyarp.IProxyArpService;
 import net.onrc.onos.ofcontroller.proxyarp.BroadcastPacketOutNotification;
+import net.onrc.onos.ofcontroller.proxyarp.IProxyArpService;
 import net.onrc.onos.ofcontroller.topology.TopologyManager;
 import net.onrc.onos.ofcontroller.util.CallerId;
 import net.onrc.onos.ofcontroller.util.DataPath;
 import net.onrc.onos.ofcontroller.util.Dpid;
+import net.onrc.onos.ofcontroller.util.FlowEntry;
 import net.onrc.onos.ofcontroller.util.FlowEntryMatch;
 import net.onrc.onos.ofcontroller.util.FlowId;
 import net.onrc.onos.ofcontroller.util.FlowPath;
@@ -59,8 +60,8 @@
 
 	private final int IDLE_TIMEOUT = 5; // seconds
 	private final int HARD_TIMEOUT = 0; // seconds
-
-	private final int PATH_PUSHED_TIMEOUT = 3000; // milliseconds
+	
+	private final CallerId callerId = new CallerId("Forwarding");
 	
 	private IFloodlightProviderService floodlightProvider;
 	private IFlowService flowService;
@@ -70,7 +71,6 @@
 	private IDeviceStorage deviceStorage;
 	private TopologyManager topologyService;
 	
-	//private Map<Path, Long> pendingFlows;
 	// TODO it seems there is a Guava collection that will time out entries.
 	// We should see if this will work here.
 	private Map<Path, PushedFlow> pendingFlows;
@@ -90,31 +90,18 @@
 	
 	private class PushedFlow {
 		public final long flowId;
-		private final long pushedTime;
-		public short firstHopOutPort = OFPort.OFPP_NONE.getValue();
+		public boolean installed = false;
 		
 		public PushedFlow(long flowId) {
 			this.flowId = flowId;
-			pushedTime = System.currentTimeMillis();
-		}
-		
-		public boolean isExpired() {
-			return (System.currentTimeMillis() - pushedTime) > PATH_PUSHED_TIMEOUT;
 		}
 	}
 	
 	private final class Path {
-		public final SwitchPort srcPort;
-		public final SwitchPort dstPort;
 		public final MACAddress srcMac;
 		public final MACAddress dstMac;
 		
-		public Path(SwitchPort src, SwitchPort dst, 
-				MACAddress srcMac, MACAddress dstMac) {
-			srcPort = new SwitchPort(new Dpid(src.dpid().value()), 
-					new Port(src.port().value()));
-			dstPort = new SwitchPort(new Dpid(dst.dpid().value()), 
-					new Port(dst.port().value()));
+		public Path(MACAddress srcMac, MACAddress dstMac) {
 			this.srcMac = srcMac;
 			this.dstMac = dstMac;
 		}
@@ -126,17 +113,13 @@
 			}
 			
 			Path otherPath = (Path) other;
-			return srcPort.equals(otherPath.srcPort) && 
-					dstPort.equals(otherPath.dstPort) &&
-					srcMac.equals(otherPath.srcMac) &&
+			return srcMac.equals(otherPath.srcMac) &&
 					dstMac.equals(otherPath.dstMac);
 		}
 		
 		@Override
 		public int hashCode() {
 			int hash = 17;
-			hash = 31 * hash + srcPort.hashCode();
-			hash = 31 * hash + dstPort.hashCode();
 			hash = 31 * hash + srcMac.hashCode();
 			hash = 31 * hash + dstMac.hashCode();
 			return hash;
@@ -144,8 +127,7 @@
 		
 		@Override
 		public String toString() {
-			return "(" + srcMac + " at " + srcPort + ") => (" 
-					+ dstPort + " at " + dstMac + ")";
+			return "(" + srcMac + ") => (" + dstMac + ")";
 		}
 	}
 	
@@ -188,12 +170,8 @@
 		datagrid = context.getServiceImpl(IDatagridService.class);
 		
 		floodlightProvider.addOFMessageListener(OFType.PACKET_IN, this);
-		
-		//pendingFlows = new ConcurrentHashMap<Path, Long>();
+
 		pendingFlows = new HashMap<Path, PushedFlow>();
-		//waitingPackets = Multimaps.synchronizedSetMultimap(
-				//HashMultimap.<Long, PacketToPush>create());
-		//waitingPackets = HashMultimap.create();
 		waitingPackets = LinkedListMultimap.create();
 		
 		deviceStorage = new DeviceStorageImpl();
@@ -243,7 +221,6 @@
 		
 		if (eth.isBroadcast() || eth.isMulticast()) {
 			handleBroadcast(sw, pi, eth);
-			//return Command.CONTINUE;
 		}
 		else {
 			// Unicast
@@ -297,26 +274,50 @@
 		MACAddress srcMacAddress = MACAddress.valueOf(eth.getSourceMACAddress());
 		MACAddress dstMacAddress = MACAddress.valueOf(eth.getDestinationMACAddress());
 		
-		
 		FlowPath flowPath, reverseFlowPath;
 		
-		Path pathspec = new Path(srcSwitchPort, dstSwitchPort, 
-				srcMacAddress, dstMacAddress);
+		Path pathspec = new Path(srcMacAddress, dstMacAddress);
 		// TODO check concurrency
 		synchronized (lock) {
 			PushedFlow existingFlow = pendingFlows.get(pathspec);
-			//Long existingFlowId = pendingFlows.get(pathspec);
-			
-			if (existingFlow != null && !existingFlow.isExpired()) {
+
+			if (existingFlow != null) {
+				// We've already installed a flow for this pair of MAC addresses
 				log.debug("Found existing flow {}", 
 						HexString.toHexString(existingFlow.flowId));
 				
 				OFPacketOut po = constructPacketOut(pi, sw);
 				
-				if (existingFlow.firstHopOutPort != OFPort.OFPP_NONE.getValue()) {
+				// Find the correct port here. We just assume the PI is from 
+				// the first hop switch, but this is definitely not always
+				// the case. We'll have to retrieve the flow from HZ every time
+				// because it could change (be rerouted) sometimes.
+				if (existingFlow.installed) {
 					// Flow has been sent to the switches so it is safe to
 					// send a packet out now
-					sendPacketOut(sw, po, existingFlow.firstHopOutPort);
+					FlowPath flow = datagrid.getFlow(new FlowId(existingFlow.flowId));
+					FlowEntry flowEntryForThisSwitch = null;
+					
+					if (flow != null) {
+						for (FlowEntry flowEntry : flow.flowEntries()) {
+							if (flowEntry.dpid().equals(new Dpid(sw.getId()))) {
+								flowEntryForThisSwitch = flowEntry;
+								break;
+							}
+						}
+					}
+					
+					if (flowEntryForThisSwitch == null) {
+						// If we don't find a flow entry for that switch, then we're
+						// in the middle of a rerouting (or something's gone wrong). 
+						// This packet will be dropped as a victim of the rerouting.
+						log.debug("Dropping packet on flow {} between {}-{}, flow path {}",
+								new Object[] {new FlowId(existingFlow.flowId),
+								srcMacAddress, dstMacAddress, flow});
+					}
+					else {
+						sendPacketOut(sw, po, flowEntryForThisSwitch.outPort().value());
+					}
 				}
 				else {
 					// Flow has not yet been sent to switches so save the
@@ -326,21 +327,16 @@
 				}
 				return;
 			}
-			
-			//log.debug("Couldn't match {} in {}", pathspec, pendingFlows);
-			
+
 			log.debug("Adding new flow between {} at {} and {} at {}",
 					new Object[]{srcMacAddress, srcSwitchPort, dstMacAddress, dstSwitchPort});
 			
-			
-			CallerId callerId = new CallerId("Forwarding");
-			
 			DataPath datapath = new DataPath();
 			datapath.setSrcPort(srcSwitchPort);
 			datapath.setDstPort(dstSwitchPort);
 			
 			flowPath = new FlowPath();
-			flowPath.setInstallerId(callerId);
+			flowPath.setInstallerId(new CallerId(callerId));
 	
 			flowPath.setFlowPathType(FlowPathType.FP_TYPE_SHORTEST_PATH);
 			flowPath.setFlowPathUserState(FlowPathUserState.FP_USER_ADD);
@@ -360,7 +356,7 @@
 			
 			// TODO implement copy constructor for FlowPath
 			reverseFlowPath = new FlowPath();
-			reverseFlowPath.setInstallerId(callerId);
+			reverseFlowPath.setInstallerId(new CallerId(callerId));
 			reverseFlowPath.setFlowPathType(FlowPathType.FP_TYPE_SHORTEST_PATH);
 			reverseFlowPath.setFlowPathUserState(FlowPathUserState.FP_USER_ADD);
 			reverseFlowPath.setIdleTimeout(IDLE_TIMEOUT);
@@ -372,9 +368,7 @@
 			reverseFlowPath.flowEntryMatch().enableEthernetFrameType(Ethernet.TYPE_IPv4);
 			reverseFlowPath.setDataPath(reverseDataPath);
 			reverseFlowPath.dataPath().srcPort().dpid().toString();
-			
-			// TODO what happens if no path exists? cleanup
-			
+
 			FlowId flowId = new FlowId(flowService.getNextFlowEntryId());
 			FlowId reverseFlowId = new FlowId(flowService.getNextFlowEntryId());
 			
@@ -382,50 +376,23 @@
 			reverseFlowPath.setFlowId(reverseFlowId);
 			
 			OFPacketOut po = constructPacketOut(pi, sw);
-			Path reversePathSpec = new Path(dstSwitchPort, srcSwitchPort, 
-					dstMacAddress, srcMacAddress);
+			Path reversePathSpec = new Path(dstMacAddress, srcMacAddress);
 			
 			// Add to waiting lists
-			//pendingFlows.put(pathspec, flowId.value());
-			//pendingFlows.put(reversePathSpec, reverseFlowId.value());
 			pendingFlows.put(pathspec, new PushedFlow(flowId.value()));
 			pendingFlows.put(reversePathSpec, new PushedFlow(reverseFlowId.value()));
 			waitingPackets.put(flowId.value(), new PacketToPush(po, sw.getId()));
 		
 		}
 		
+		log.debug("Adding reverse {} to {} flowid {}", new Object[] {
+				dstMacAddress, srcMacAddress, reverseFlowPath.flowId()});
 		flowService.addFlow(reverseFlowPath);
+		log.debug("Adding forward {} to {} flowid {}", new Object[] {
+				srcMacAddress, dstMacAddress, flowPath.flowId()});
 		flowService.addFlow(flowPath);
 		
 	}
-	
-	/*
-	private boolean flowExists(SwitchPort srcPort, MACAddress srcMac, 
-			SwitchPort dstPort, MACAddress dstMac) {
-		for (FlowPath flow : datagridService.getAllFlows()) {
-			FlowEntryMatch match = flow.flowEntryMatch();
-			// TODO implement FlowEntryMatch.equals();
-			// This is painful to do properly without support in the FlowEntryMatch
-			boolean same = true;
-			if (!match.srcMac().equals(srcMac) ||
-				!match.dstMac().equals(dstMac)) {
-				same = false;
-			}
-			if (!flow.dataPath().srcPort().equals(srcPort) || 
-				!flow.dataPath().dstPort().equals(dstPort)) {
-				same = false;
-			}
-			
-			if (same) {
-				log.debug("found flow entry that's the same {}-{}:::{}-{}",
-						new Object[] {srcPort, srcMac, dstPort, dstMac});
-				return true;
-			}
-		}
-		
-		return false;
-	}
-	*/
 
 	private OFPacketOut constructPacketOut(OFPacketIn pi, IOFSwitch sw) {	
 		OFPacketOut po = new OFPacketOut();
@@ -452,36 +419,64 @@
 			flowInstalled(flowPath);
 		}
 	}
+	
+	@Override
+	public void flowRemoved(FlowPath removedFlowPath) {
+		if (!removedFlowPath.installerId().equals(callerId)) {
+			// Not our flow path, ignore
+			return;
+		}
+		
+		MACAddress srcMacAddress = removedFlowPath.flowEntryMatch().srcMac();
+		MACAddress dstMacAddress = removedFlowPath.flowEntryMatch().dstMac();
+		
+		Path removedPath = new Path(srcMacAddress, dstMacAddress);
+		
+		synchronized (lock) {
+			pendingFlows.remove(removedPath);
+			
+			// There *shouldn't* be any packets queued if the flow has 
+			// just been removed. 
+			List<PacketToPush> packets = 
+					waitingPackets.removeAll(removedFlowPath.flowId().value());
+			if (!packets.isEmpty()) {
+				log.warn("Removed flow {} has packets queued", 
+						removedFlowPath.flowId());
+			}
+		}
+	}
 
 	private void flowInstalled(FlowPath installedFlowPath) {
 		long flowId = installedFlowPath.flowId().value();
 		
+		if (!installedFlowPath.installerId().equals(callerId)) {
+			// Not our flow path, ignore
+			return;
+		}
+		
+		// TODO waiting packets should time out. We could request a path that
+		// can't be installed right now because of a network partition. The path
+		// may eventually be installed, but we may have received thousands of 
+		// packets in the meantime and probably don't want to send very old packets.
 		short outPort = 
 				installedFlowPath.flowEntries().get(0).outPort().value();
 		
 		MACAddress srcMacAddress = installedFlowPath.flowEntryMatch().srcMac();
 		MACAddress dstMacAddress = installedFlowPath.flowEntryMatch().dstMac();
 		
-		if (srcMacAddress == null || dstMacAddress == null) {
-			// Not our flow path, ignore
-			return;
-		}
-		
 		Collection<PacketToPush> packets;
 		synchronized (lock) {
-			log.debug("Flow {} has been installed, sending queued packets",
-					installedFlowPath.flowId());
-			
 			packets = waitingPackets.removeAll(flowId);
 			
+			log.debug("Flow {} has been installed, sending {} queued packets",
+					installedFlowPath.flowId(), packets.size());
+			
 			// remove pending flows entry
-			Path installedPath = new Path(installedFlowPath.dataPath().srcPort(),
-					installedFlowPath.dataPath().dstPort(),
-					srcMacAddress, dstMacAddress);
-			//pendingFlows.remove(pathToRemove);
+			Path installedPath = new Path(srcMacAddress, dstMacAddress);
 			PushedFlow existingFlow = pendingFlows.get(installedPath);
-			if (existingFlow != null)
-			    existingFlow.firstHopOutPort = outPort;
+			if (existingFlow != null) {
+			    existingFlow.installed = true;
+			}
 		}
 		
 		for (PacketToPush packet : packets) {
@@ -499,4 +494,5 @@
 		
 		flowPusher.add(sw, po);
 	}
+
 }
diff --git a/src/main/java/net/onrc/onos/ofcontroller/forwarding/IForwardingService.java b/src/main/java/net/onrc/onos/ofcontroller/forwarding/IForwardingService.java
index e5bd714..0e0d1da 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/forwarding/IForwardingService.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/forwarding/IForwardingService.java
@@ -22,4 +22,12 @@
 	 * been installed in the network.
 	 */
 	public void flowsInstalled(Collection<FlowPath> installedFlowPaths);
+	
+	/**
+	 * Notify the Forwarding module that a flow has expired and been 
+	 * removed from the network.
+	 * 
+	 * @param removedFlowPath The FlowPath that was removed
+	 */
+	public void flowRemoved(FlowPath removedFlowPath);
 }
diff --git a/src/main/java/net/onrc/onos/ofcontroller/util/CallerId.java b/src/main/java/net/onrc/onos/ofcontroller/util/CallerId.java
index 0607533..a0217d4 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/util/CallerId.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/util/CallerId.java
@@ -12,6 +12,15 @@
      * Default constructor.
      */
     public CallerId() {}
+    
+    /**
+     * Copy constructor
+     * @param otherCallerId
+     */
+    public CallerId(CallerId otherCallerId) {
+    // Note: make a full copy if we change value to a mutable type
+    value = otherCallerId.value;
+    }
 
     /**
      * Constructor from a string value.
@@ -49,4 +58,20 @@
     public String toString() {
 	return value;
     }
+    
+    @Override
+    public boolean equals(Object other) {
+    if (!(other instanceof CallerId)) {
+        return false;
+    }
+    
+    CallerId otherCallerId = (CallerId) other;
+    
+    return value.equals(otherCallerId.value);
+    }
+    
+    @Override
+    public int hashCode() {
+    return value.hashCode();
+    }
 }
diff --git a/src/main/java/net/onrc/onos/ofcontroller/util/serializers/KryoFactory.java b/src/main/java/net/onrc/onos/ofcontroller/util/serializers/KryoFactory.java
index 883f830..5998dcd 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/util/serializers/KryoFactory.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/util/serializers/KryoFactory.java
@@ -30,7 +30,7 @@
 import net.onrc.onos.ofcontroller.util.Switch;
 import net.onrc.onos.ofcontroller.util.SwitchPort;
 
-import com.esotericsoftware.kryo2.Kryo;
+import com.esotericsoftware.kryo.Kryo;
 
 /**
  * Class factory for allocating Kryo instances for