Work toward fixing Bug ONOS-859:
If there is a Titan exception when adding or deleting Flows to the GraphDB,
keep retrying until success.
NOTE: Most likely this issue is NOT the root cause for (most of)
the missing flows as reported by Bug ONOS-859.
Initial investigation on 4-node VM setup in the lab couldn't reproduce
the problem. We need to investigate the root cause on the 3-node
Baremetal cluster after it becomes available.
Reviewed by: Pankaj
diff --git a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
index 88a0cbf..b942aae 100644
--- a/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
+++ b/src/main/java/net/onrc/onos/ofcontroller/flowmanager/FlowManager.java
@@ -37,6 +37,8 @@
import net.onrc.onos.ofcontroller.util.Pair;
import net.onrc.onos.ofcontroller.util.serializers.KryoFactory;
+import com.thinkaurelius.titan.core.TitanException;
+
import com.esotericsoftware.kryo2.Kryo;
import org.slf4j.Logger;
@@ -669,16 +671,25 @@
log.debug("Deleting Flow Path From Database: {}",
flowPath.toString());
- try {
- if (! FlowDatabaseOperation.deleteFlow(
- dbHandlerInner,
- flowPath.flowId())) {
- log.error("Cannot delete Flow Path {} from Network Map",
- flowPath.flowId());
- }
- } catch (Exception e) {
- log.error("Exception deleting Flow Path from Network MAP: {}", e);
- }
+ boolean retry = false;
+ do {
+ retry = false;
+ try {
+ if (! FlowDatabaseOperation.deleteFlow(
+ dbHandlerInner,
+ flowPath.flowId())) {
+ log.error("Cannot delete Flow Path {} from Network Map",
+ flowPath.flowId());
+ retry = true;
+ }
+ } catch (TitanException te) {
+ log.error("Titan Exception deleting Flow Path from Network MAP: {}", te);
+ retry = true;
+ } catch (Exception e) {
+ log.error("Exception deleting Flow Path from Network MAP: {}", e);
+ }
+ } while (retry);
+
continue;
}
@@ -709,15 +720,23 @@
//
// Write the Flow Path to the Network Map
//
- try {
- if (! FlowDatabaseOperation.addFlow(dbHandlerInner, flowPath)) {
- String logMsg = "Cannot write to Network Map Flow Path " +
- flowPath.flowId();
- log.error(logMsg);
+ boolean retry = false;
+ do {
+ retry = false;
+ try {
+ if (! FlowDatabaseOperation.addFlow(dbHandlerInner, flowPath)) {
+ String logMsg = "Cannot write to Network Map Flow Path " +
+ flowPath.flowId();
+ log.error(logMsg);
+ retry = true;
+ }
+ } catch (TitanException te) {
+ log.error("Titan Exception writing Flow Path to Network MAP: ", te);
+ retry = true;
+ } catch (Exception e) {
+ log.error("Exception writing Flow Path to Network MAP: ", e);
}
- } catch (Exception e) {
- log.error("Exception writing Flow Path to Network MAP: ", e);
- }
+ } while (retry);
}
}
}