ONOS-3763 Change flow state to PENDING_ADD when retrying

Will emit a RULE_UPDATE event if the state is changed.
Update unit test accordingly.

Change-Id: Ie84778c62f52f15b7636d41db246814145e73f77
diff --git a/core/api/src/main/java/org/onosproject/net/flow/FlowRuleStore.java b/core/api/src/main/java/org/onosproject/net/flow/FlowRuleStore.java
index d81c73c..ae20f2a 100644
--- a/core/api/src/main/java/org/onosproject/net/flow/FlowRuleStore.java
+++ b/core/api/src/main/java/org/onosproject/net/flow/FlowRuleStore.java
@@ -97,6 +97,16 @@
     FlowRuleEvent removeFlowRule(FlowEntry rule);
 
     /**
+     * Marks a flow rule as PENDING_ADD during retry.
+     *
+     * Emits flow_update event if the state is changed
+     *
+     * @param rule the flow rule that is retrying
+     * @return flow_updated event, or null if nothing updated
+     */
+    FlowRuleEvent pendingFlowRule(FlowEntry rule);
+
+    /**
      * Updates the flow table statistics of the specified device using
      * the given statistics.
      *
diff --git a/core/common/src/test/java/org/onosproject/store/trivial/SimpleFlowRuleStore.java b/core/common/src/test/java/org/onosproject/store/trivial/SimpleFlowRuleStore.java
index cb86fd5..33f1cc5 100644
--- a/core/common/src/test/java/org/onosproject/store/trivial/SimpleFlowRuleStore.java
+++ b/core/common/src/test/java/org/onosproject/store/trivial/SimpleFlowRuleStore.java
@@ -258,6 +258,23 @@
     }
 
     @Override
+    public FlowRuleEvent pendingFlowRule(FlowEntry rule) {
+        List<StoredFlowEntry> entries = getFlowEntries(rule.deviceId(), rule.id());
+        synchronized (entries) {
+            for (StoredFlowEntry entry : entries) {
+                if (entry.equals(rule) &&
+                        entry.state() != FlowEntryState.PENDING_ADD) {
+                    synchronized (entry) {
+                        entry.setState(FlowEntryState.PENDING_ADD);
+                        return new FlowRuleEvent(Type.RULE_UPDATED, rule);
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
     public void storeBatch(
             FlowRuleBatchOperation operation) {
         List<FlowRuleBatchEntry> toAdd = new ArrayList<>();
diff --git a/core/net/src/main/java/org/onosproject/net/flow/impl/FlowRuleManager.java b/core/net/src/main/java/org/onosproject/net/flow/impl/FlowRuleManager.java
index de92894..570383a 100644
--- a/core/net/src/main/java/org/onosproject/net/flow/impl/FlowRuleManager.java
+++ b/core/net/src/main/java/org/onosproject/net/flow/impl/FlowRuleManager.java
@@ -305,6 +305,7 @@
                     break;
                 case ADDED:
                 case PENDING_ADD:
+                    event = store.pendingFlowRule(flowRule);
                     try {
                         frp.applyFlowRule(flowRule);
                     } catch (UnsupportedOperationException e) {
diff --git a/core/net/src/test/java/org/onosproject/net/flow/impl/FlowRuleManagerTest.java b/core/net/src/test/java/org/onosproject/net/flow/impl/FlowRuleManagerTest.java
index 553745b..24ade7e 100644
--- a/core/net/src/test/java/org/onosproject/net/flow/impl/FlowRuleManagerTest.java
+++ b/core/net/src/test/java/org/onosproject/net/flow/impl/FlowRuleManagerTest.java
@@ -206,7 +206,7 @@
         assertEquals("should still be 2 rules", 2, flowCount());
 
         providerService.pushFlowMetrics(DID, ImmutableList.of(fe1));
-        validateEvents(RULE_UPDATED);
+        validateEvents(RULE_UPDATED, RULE_UPDATED);
     }
 
     private boolean validateState(Map<FlowRule, FlowEntryState> expected) {
@@ -293,7 +293,7 @@
         service.applyFlowRules(f3);
 
         providerService.pushFlowMetrics(DID, Collections.singletonList(fe3));
-        validateEvents(RULE_ADD_REQUESTED, RULE_ADDED);
+        validateEvents(RULE_ADD_REQUESTED, RULE_ADDED, RULE_UPDATED);
 
         providerService.flowRemoved(fe3);
         validateEvents();
diff --git a/core/store/dist/src/main/java/org/onosproject/store/flow/impl/NewDistributedFlowRuleStore.java b/core/store/dist/src/main/java/org/onosproject/store/flow/impl/NewDistributedFlowRuleStore.java
index 99dec1d..662a7eb 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/flow/impl/NewDistributedFlowRuleStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/flow/impl/NewDistributedFlowRuleStore.java
@@ -526,6 +526,19 @@
     }
 
     @Override
+    public FlowRuleEvent pendingFlowRule(FlowEntry rule) {
+        if (mastershipService.isLocalMaster(rule.deviceId())) {
+            StoredFlowEntry stored = flowTable.getFlowEntry(rule);
+            if (stored != null &&
+                    stored.state() != FlowEntryState.PENDING_ADD) {
+                stored.setState(FlowEntryState.PENDING_ADD);
+                return new FlowRuleEvent(Type.RULE_UPDATED, rule);
+            }
+        }
+        return null;
+    }
+
+    @Override
     public FlowRuleEvent addOrUpdateFlowRule(FlowEntry rule) {
         NodeId master = mastershipService.getMasterFor(rule.deviceId());
         if (Objects.equals(local, master)) {