* Refactor/cleanup the processing of events inside the FlowEventHandler.
* Refactor/cleanup the pushing of the modified Flow Entries inside
the FlowManager.
diff --git a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowEventHandler.java b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowEventHandler.java
index cb1e678..f3e47f4 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowEventHandler.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowEventHandler.java
@@ -10,6 +10,7 @@
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import net.floodlightcontroller.core.IOFSwitch;
import net.onrc.onos.datagrid.IDatagridService;
import net.onrc.onos.ofcontroller.topology.Topology;
import net.onrc.onos.ofcontroller.topology.TopologyElement;
@@ -60,6 +61,17 @@
private List<EventEntry<FlowEntry>> flowEntryEvents =
new LinkedList<EventEntry<FlowEntry>>();
+ //
+ // Transient state for processing the Flow Paths:
+ // - The new Flow Paths
+ // - The Flow Paths that should be recomputed
+ // - The Flow Paths with modified Flow Entries
+ //
+ private List<FlowPath> newFlowPaths = new LinkedList<FlowPath>();
+ private List<FlowPath> recomputeFlowPaths = new LinkedList<FlowPath>();
+ private List<FlowPath> modifiedFlowPaths = new LinkedList<FlowPath>();
+
+
/**
* Constructor for a given Flow Manager and Datagrid Service.
*
@@ -81,10 +93,9 @@
protected Topology getTopology() { return this.topology; }
/**
- * Run the thread.
+ * Startup processing.
*/
- @Override
- public void run() {
+ private void startup() {
//
// Obtain the initial Topology state
//
@@ -116,6 +127,14 @@
// Process the initial events (if any)
processEvents();
+ }
+
+ /**
+ * Run the thread.
+ */
+ @Override
+ public void run() {
+ startup();
//
// The main loop
@@ -162,17 +181,108 @@
* Process the events (if any)
*/
private void processEvents() {
- List<FlowPath> newFlowPaths = new LinkedList<FlowPath>();
- List<FlowPath> recomputeFlowPaths = new LinkedList<FlowPath>();
- List<FlowPath> modifiedFlowPaths = new LinkedList<FlowPath>();
+ List<FlowEntry> modifiedFlowEntries;
if (topologyEvents.isEmpty() && flowPathEvents.isEmpty() &&
flowEntryEvents.isEmpty()) {
return; // Nothing to do
}
+ processFlowPathEvents();
+ processTopologyEvents();
//
- // Process the Flow Path events
+ // Add all new Flows: should be done after processing the Flow Path
+ // and Topology events.
+ //
+ for (FlowPath flowPath : newFlowPaths) {
+ allFlowPaths.put(flowPath.flowId().value(), flowPath);
+ }
+
+ processFlowEntryEvents();
+
+ // Recompute all affected Flow Paths and keep only the modified
+ for (FlowPath flowPath : recomputeFlowPaths) {
+ if (recomputeFlowPath(flowPath))
+ modifiedFlowPaths.add(flowPath);
+ }
+
+ modifiedFlowEntries = extractModifiedFlowEntries(modifiedFlowPaths);
+
+ // Assign missing Flow Entry IDs
+ assignFlowEntryId(modifiedFlowEntries);
+
+ //
+ // Push the modified Flow Entries to switches, datagrid and database
+ //
+ flowManager.pushModifiedFlowEntriesToSwitches(modifiedFlowPaths);
+ flowManager.pushModifiedFlowEntriesToDatagrid(modifiedFlowEntries);
+ flowManager.pushModifiedFlowEntriesToDatabase(modifiedFlowEntries);
+
+ //
+ // Remove Flow Entries that were deleted
+ //
+ for (FlowPath flowPath : modifiedFlowPaths)
+ flowPath.dataPath().removeDeletedFlowEntries();
+
+ // Cleanup
+ topologyEvents.clear();
+ flowPathEvents.clear();
+ flowEntryEvents.clear();
+ //
+ newFlowPaths.clear();
+ recomputeFlowPaths.clear();
+ modifiedFlowPaths.clear();
+ }
+
+ /**
+ * Extract the modified Flow Entries.
+ */
+ private List<FlowEntry> extractModifiedFlowEntries(
+ List<FlowPath> modifiedFlowPaths) {
+ List<FlowEntry> modifiedFlowEntries = new LinkedList<FlowEntry>();
+
+ // Extract only the modified Flow Entries
+ for (FlowPath flowPath : modifiedFlowPaths) {
+ for (FlowEntry flowEntry : flowPath.flowEntries()) {
+ if (flowEntry.flowEntrySwitchState() ==
+ FlowEntrySwitchState.FE_SWITCH_NOT_UPDATED) {
+ modifiedFlowEntries.add(flowEntry);
+ }
+ }
+ }
+ return modifiedFlowEntries;
+ }
+
+ /**
+ * Assign the Flow Entry ID as needed.
+ */
+ private void assignFlowEntryId(List<FlowEntry> modifiedFlowEntries) {
+ if (modifiedFlowEntries.isEmpty())
+ return;
+
+ Map<Long, IOFSwitch> mySwitches = flowManager.getMySwitches();
+
+ //
+ // Assign the Flow Entry ID only for Flow Entries for my switches
+ //
+ for (FlowEntry flowEntry : modifiedFlowEntries) {
+ // Update the Flow Entries only for my switches
+ IOFSwitch mySwitch = mySwitches.get(flowEntry.dpid().value());
+ if (mySwitch == null)
+ continue;
+ if (! flowEntry.isValidFlowEntryId()) {
+ long id = flowManager.getNextFlowEntryId();
+ flowEntry.setFlowEntryId(new FlowEntryId(id));
+ }
+ }
+ }
+
+ /**
+ * Process the Flow Path events.
+ */
+ private void processFlowPathEvents() {
+ //
+ // Process all Flow Path events and update the appropriate state
//
for (EventEntry<FlowPath> eventEntry : flowPathEvents) {
FlowPath flowPath = eventEntry.eventData();
@@ -243,9 +353,14 @@
}
}
}
+ }
+ /**
+ * Process the Topology events.
+ */
+ private void processTopologyEvents() {
//
- // Process the topology events
+ // Process all Topology events and update the appropriate state
//
boolean isTopologyModified = false;
for (EventEntry<TopologyElement> eventEntry : topologyEvents) {
@@ -267,22 +382,17 @@
// TODO: For now, if the topology changes, we recompute all Flows
recomputeFlowPaths.addAll(allFlowPaths.values());
}
+ }
- // Add all new Flows
- for (FlowPath flowPath : newFlowPaths) {
- allFlowPaths.put(flowPath.flowId().value(), flowPath);
- }
-
- // Recompute all affected Flow Paths and keep only the modified
- for (FlowPath flowPath : recomputeFlowPaths) {
- if (recomputeFlowPath(flowPath))
- modifiedFlowPaths.add(flowPath);
- }
-
+ /**
+ * Process the Flow Entry events.
+ */
+ private void processFlowEntryEvents() {
//
- // Process previously unmatched Flow Entry updates
+ // Update Flow Entries with previously unmatched Flow Entry updates
//
- if ((! flowPathEvents.isEmpty()) && (! unmatchedFlowEntryUpdates.isEmpty())) {
+ if ((! flowPathEvents.isEmpty()) &&
+ (! unmatchedFlowEntryUpdates.isEmpty())) {
List<FlowEntry> remainingUpdates = new LinkedList<FlowEntry>();
for (FlowEntry flowEntry : unmatchedFlowEntryUpdates) {
if (! updateFlowEntry(flowEntry))
@@ -292,7 +402,7 @@
}
//
- // Process the Flow Entry events
+ // Process all Flow Entry events and update the appropriate state
//
for (EventEntry<FlowEntry> eventEntry : flowEntryEvents) {
FlowEntry flowEntry = eventEntry.eventData();
@@ -315,16 +425,6 @@
break;
}
}
-
- //
- // Push the Flow Entries that have been modified
- //
- flowManager.pushModifiedFlowEntries(modifiedFlowPaths);
-
- // Cleanup
- topologyEvents.clear();
- flowPathEvents.clear();
- flowEntryEvents.clear();
}
/**
diff --git a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
index 98f9cce..415e281 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
@@ -718,6 +718,15 @@
}
/**
+ * Get the collection of my switches.
+ *
+ * @return the collection of my switches.
+ */
+ public Map<Long, IOFSwitch> getMySwitches() {
+ return floodlightProvider.getSwitches();
+ }
+
+ /**
* Get the network topology.
*
* @return the network topology.
@@ -855,16 +864,16 @@
}
/**
- * Push the modified Flow Entries of a collection of Flow Paths.
- * Only the Flow Entries to switches controlled by this instance
- * are pushed.
+ * Push modified Flow Entries to switches.
*
- * NOTE: Currently, we write to both the Network MAP and the switches.
+ * NOTE: Only the Flow Entries to switches controlled by this instance
+ * are pushed.
*
* @param modifiedFlowPaths the collection of Flow Paths with the modified
* Flow Entries.
*/
- public void pushModifiedFlowEntries(Collection<FlowPath> modifiedFlowPaths) {
+ public void pushModifiedFlowEntriesToSwitches(
+ Collection<FlowPath> modifiedFlowPaths) {
// TODO: For now, the pushing of Flow Entries is disabled
if (true)
return;
@@ -872,17 +881,9 @@
if (modifiedFlowPaths.isEmpty())
return;
- Map<Long, IOFSwitch> mySwitches = floodlightProvider.getSwitches();
+ Map<Long, IOFSwitch> mySwitches = getMySwitches();
for (FlowPath flowPath : modifiedFlowPaths) {
- //
- // Find the Flow Path in the Network MAP.
- // NOTE: The Flow Path might not be found if the Flow was just
- // removed by some other controller instance.
- //
- IFlowPath flowObj = dbHandlerInner.searchFlowPath(flowPath.flowId());
-
- boolean isFlowEntryDeleted = false;
for (FlowEntry flowEntry : flowPath.flowEntries()) {
log.debug("Updating Flow Entry: {}", flowEntry.toString());
@@ -890,91 +891,160 @@
FlowEntrySwitchState.FE_SWITCH_NOT_UPDATED) {
continue; // No need to update the entry
}
- if (flowEntry.flowEntryUserState() ==
- FlowEntryUserState.FE_USER_DELETE) {
- isFlowEntryDeleted = true;
- }
- //
- // Install the Flow Entries into my switches
- //
IOFSwitch mySwitch = mySwitches.get(flowEntry.dpid().value());
- if (mySwitch != null) {
- //
- // Assign the FlowEntry ID if needed
- //
- if (! flowEntry.isValidFlowEntryId()) {
- long id = getNextFlowEntryId();
- flowEntry.setFlowEntryId(new FlowEntryId(id));
- }
-
- //
- // Install the Flow Entry into the switch
- //
- if (! installFlowEntry(mySwitch, flowPath, flowEntry)) {
- String logMsg = "Cannot install Flow Entry " +
- flowEntry.flowEntryId() +
- " from Flow Path " + flowPath.flowId() +
- " on switch " + flowEntry.dpid();
- log.error(logMsg);
- continue;
- }
-
- //
- // NOTE: Here we assume that the switch has been
- // successfully updated.
- //
- flowEntry.setFlowEntrySwitchState(FlowEntrySwitchState.FE_SWITCH_UPDATED);
- }
+ if (mySwitch == null)
+ continue;
//
- // TODO: For now Flow Entries are removed from the Datagrid
- // and from the Network Map by all instances, even if this
- // Flow Entry is not for our switches.
+ // Install the Flow Entry into the switch
//
- // This is needed to handle the case a switch going down:
- // it has no Master controller instance, hence no
- // controller instance will cleanup its flow entries.
- // This is sub-optimal: we need to elect a controller
- // instance to handle the cleanup of such orphaned flow
- // entries.
- //
- if (mySwitch == null) {
- if (flowEntry.flowEntryUserState() !=
- FlowEntryUserState.FE_USER_DELETE) {
- continue;
- }
- if (! flowEntry.isValidFlowEntryId())
- continue;
- }
-
- //
- // Write the Flow Entry to the Datagrid
- //
- switch (flowEntry.flowEntryUserState()) {
- case FE_USER_ADD:
- if (mySwitch == null)
- break; // Install only flow entries for my switches
- datagridService.notificationSendFlowEntryAdded(flowEntry);
- break;
- case FE_USER_MODIFY:
- if (mySwitch == null)
- break; // Install only flow entries for my switches
- datagridService.notificationSendFlowEntryUpdated(flowEntry);
- break;
- case FE_USER_DELETE:
- datagridService.notificationSendFlowEntryRemoved(flowEntry.flowEntryId());
- break;
- }
-
- //
- // Write the Flow Entry to the Network Map
- //
- if (flowObj == null) {
- String logMsg = "Cannot find Network MAP entry for Flow Path " + flowPath.flowId();
+ if (! installFlowEntry(mySwitch, flowPath, flowEntry)) {
+ String logMsg = "Cannot install Flow Entry " +
+ flowEntry.flowEntryId() +
+ " from Flow Path " + flowPath.flowId() +
+ " on switch " + flowEntry.dpid();
+ log.error(logMsg);
continue;
}
+
+ //
+ // NOTE: Here we assume that the switch has been
+ // successfully updated.
+ //
+ flowEntry.setFlowEntrySwitchState(FlowEntrySwitchState.FE_SWITCH_UPDATED);
+ }
+ }
+ }
+
+ /**
+ * Push modified Flow Entries to the datagrid.
+ *
+ * @param modifiedFlowEntries the collection of modified Flow Entries.
+ */
+ public void pushModifiedFlowEntriesToDatagrid(
+ Collection<FlowEntry> modifiedFlowEntries) {
+ // TODO: For now, the pushing of Flow Entries is disabled
+ if (true)
+ return;
+
+ if (modifiedFlowEntries.isEmpty())
+ return;
+
+ Map<Long, IOFSwitch> mySwitches = getMySwitches();
+
+ for (FlowEntry flowEntry : modifiedFlowEntries) {
+ IOFSwitch mySwitch = mySwitches.get(flowEntry.dpid().value());
+
+ //
+ // TODO: For now Flow Entries are removed by all instances,
+ // even if this Flow Entry is not for our switches.
+ //
+ // This is needed to handle the case a switch going down:
+ // it has no Master controller instance, hence no
+ // controller instance will cleanup its flow entries.
+ // This is sub-optimal: we need to elect a controller
+ // instance to handle the cleanup of such orphaned flow
+ // entries.
+ //
+ if (mySwitch == null) {
+ if (flowEntry.flowEntryUserState() !=
+ FlowEntryUserState.FE_USER_DELETE) {
+ continue;
+ }
+ if (! flowEntry.isValidFlowEntryId())
+ continue;
+ }
+
+ //
+ // Write the Flow Entry to the Datagrid
+ //
+ switch (flowEntry.flowEntryUserState()) {
+ case FE_USER_ADD:
+ if (mySwitch == null)
+ break; // Install only flow entries for my switches
+ datagridService.notificationSendFlowEntryAdded(flowEntry);
+ break;
+ case FE_USER_MODIFY:
+ if (mySwitch == null)
+ break; // Install only flow entries for my switches
+ datagridService.notificationSendFlowEntryUpdated(flowEntry);
+ break;
+ case FE_USER_DELETE:
+ datagridService.notificationSendFlowEntryRemoved(flowEntry.flowEntryId());
+ break;
+ }
+ }
+ }
+
+ /**
+ * Push Flow Entries to the Network MAP.
+ *
+ * @param modifiedFlowEntries the collection of Flow Entries to push.
+ */
+ public void pushModifiedFlowEntriesToDatabase(
+ Collection<FlowEntry> modifiedFlowEntries) {
+ // TODO: For now, the pushing of Flow Entries is disabled
+ if (true)
+ return;
+
+ if (modifiedFlowEntries.isEmpty())
+ return;
+
+ Map<Long, IOFSwitch> mySwitches = getMySwitches();
+
+ for (FlowEntry flowEntry : modifiedFlowEntries) {
+ if (! flowEntry.isValidFlowId()) {
+ // Shouldn't happen
+ log.error("Cannot push Flow Entry to database: invalid Flow ID: {}", flowEntry.toString());
+ continue;
+ }
+
+ IOFSwitch mySwitch = mySwitches.get(flowEntry.dpid().value());
+
+ //
+ // TODO: For now Flow Entries are removed by all instances,
+ // even if this Flow Entry is not for our switches.
+ //
+ // This is needed to handle the case a switch going down:
+ // it has no Master controller instance, hence no
+ // controller instance will cleanup its flow entries.
+ // This is sub-optimal: we need to elect a controller
+ // instance to handle the cleanup of such orphaned flow
+ // entries.
+ //
+ if (mySwitch == null) {
+ if (flowEntry.flowEntryUserState() !=
+ FlowEntryUserState.FE_USER_DELETE) {
+ continue;
+ }
+ if (! flowEntry.isValidFlowEntryId())
+ continue;
+ }
+
+ //
+ // Write the Flow Entry to the Network Map
+ //
+ // NOTE: We try a number of times; apparently, if other instances
+ // are writing at the same time this will trigger an error.
+ //
+ for (int i = 0; i < 6; i++) {
try {
+ //
+ // Find the Flow Path in the Network MAP.
+ //
+ // NOTE: The Flow Path might not be found if the Flow was
+ // just removed by some other controller instance.
+ //
+ IFlowPath flowObj =
+ dbHandlerInner.searchFlowPath(flowEntry.flowId());
+ if (flowObj == null) {
+ String logMsg = "Cannot find Network MAP entry for Flow Path " + flowEntry.flowId();
+ log.error(logMsg);
+ break;
+ }
+
+ // Write the Flow Entry
switch (flowEntry.flowEntryUserState()) {
case FE_USER_ADD:
// FALLTHROUGH
@@ -982,7 +1052,7 @@
if (addFlowEntry(flowObj, flowEntry) == null) {
String logMsg = "Cannot write to Network MAP Flow Entry " +
flowEntry.flowEntryId() +
- " from Flow Path " + flowPath.flowId() +
+ " from Flow Path " + flowEntry.flowId() +
" on switch " + flowEntry.dpid();
log.error(logMsg);
}
@@ -991,45 +1061,28 @@
if (deleteFlowEntry(flowObj, flowEntry) == false) {
String logMsg = "Cannot remove from Network MAP Flow Entry " +
flowEntry.flowEntryId() +
- " from Flow Path " + flowPath.flowId() +
+ " from Flow Path " + flowEntry.flowId() +
" on switch " + flowEntry.dpid();
log.error(logMsg);
}
break;
}
+
+ // Commit to the database
+ dbHandlerInner.commit();
+ break; // Success
+
} catch (Exception e) {
log.debug("Exception writing Flow Entry to Network MAP: ", e);
dbHandlerInner.rollback();
- continue;
- }
- }
-
- //
- // Remove Flow Entries that were deleted
- //
- // NOTE: We create a new ArrayList, and add only the Flow Entries
- // that are NOT FE_USER_DELETE.
- // This is sub-optimal: if it adds notable processing cost,
- // the Flow Entries container should be changed to LinkedList
- // or some other container that has O(1) cost of removing an entry.
- //
- if (isFlowEntryDeleted) {
- ArrayList<FlowEntry> newFlowEntries = new ArrayList<FlowEntry>();
- for (FlowEntry flowEntry : flowPath.flowEntries()) {
- if (flowEntry.flowEntryUserState() !=
- FlowEntryUserState.FE_USER_DELETE) {
- newFlowEntries.add(flowEntry);
+ // Wait a bit (random value [1ms, 20ms] and try again
+ int delay = 1 + randomGenerator.nextInt() % 20;
+ try {
+ Thread.sleep(delay);
+ } catch (Exception e0) {
}
}
- flowPath.dataPath().setFlowEntries(newFlowEntries);
}
}
- // Try to commit to the database
- try {
- dbHandlerInner.commit();
- } catch (Exception e) {
- log.debug("Exception during commit of Flow Entries to Network MAP", e);
- dbHandlerInner.rollback();
- }
}
}