P4-related cherry-picks for 1.12
Squashed. Includes the following commits from master:
8b19a07297 Fixed onos.py
74174bf177 Fix 'unable to translate flow rule' in p4-tutorial icmpdropper
4555c5f308 Minor refactoring of BMv2 mininet scripts
07b2b70f53 Refactored PI-ECMP app to use action profiles of basic.p4
6ffd3772b7 ONOS-7050 First stab at PI translation store
2d4271fc20 ONOS-7050 Refactored PI translation service and store
3874b44821 ONOS-7050 Refactored P4Runtime FRP to use distributed stores
41efe435be ONOS-7050 Refactored P4Runtime GP to use distributed stores
806f7b7418 ONOS-6810 Implement Mastership handling in general DeviceProvider
c7922a4b40 ONOS-7267 Fix pipeconf UI
Change-Id: I279b6477f48ebec768b494799feb12faadbd559c
diff --git a/drivers/p4runtime/BUCK b/drivers/p4runtime/BUCK
index b8167e8..09c9bf7 100644
--- a/drivers/p4runtime/BUCK
+++ b/drivers/p4runtime/BUCK
@@ -2,9 +2,11 @@
COMPILE_DEPS = [
'//lib:CORE_DEPS',
+ '//lib:KRYO',
'//protocols/p4runtime/api:onos-protocols-p4runtime-api',
'//incubator/grpc-dependencies:grpc-core-repkg-' + GRPC_VER,
'//lib:grpc-netty-' + GRPC_VER,
+ '//core/store/serializers:onos-core-serializers',
]
BUNDLES = [
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
index 5f95eda..ca9392f 100644
--- a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
@@ -20,6 +20,9 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import io.grpc.StatusRuntimeException;
+import org.onlab.util.SharedExecutors;
+import org.onosproject.drivers.p4runtime.mirror.P4RuntimeTableMirror;
+import org.onosproject.drivers.p4runtime.mirror.TimedEntry;
import org.onosproject.net.flow.DefaultFlowEntry;
import org.onosproject.net.flow.FlowEntry;
import org.onosproject.net.flow.FlowRule;
@@ -32,15 +35,17 @@
import org.onosproject.net.pi.runtime.PiCounterCellData;
import org.onosproject.net.pi.runtime.PiCounterCellId;
import org.onosproject.net.pi.runtime.PiTableEntry;
-import org.onosproject.net.pi.service.PiTranslationService;
+import org.onosproject.net.pi.runtime.PiTableEntryHandle;
+import org.onosproject.net.pi.service.PiFlowRuleTranslator;
+import org.onosproject.net.pi.service.PiTranslatedEntity;
+import org.onosproject.net.pi.service.PiTranslationException;
import org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType;
-import org.onosproject.p4runtime.api.P4RuntimeFlowRuleWrapper;
-import org.onosproject.p4runtime.api.P4RuntimeTableEntryReference;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
@@ -49,6 +54,7 @@
import java.util.stream.Collectors;
import static com.google.common.collect.Lists.newArrayList;
+import static java.util.Collections.singleton;
import static org.onosproject.drivers.p4runtime.P4RuntimeFlowRuleProgrammable.Operation.APPLY;
import static org.onosproject.drivers.p4runtime.P4RuntimeFlowRuleProgrammable.Operation.REMOVE;
import static org.onosproject.net.flow.FlowEntry.FlowEntryState.ADDED;
@@ -59,44 +65,40 @@
/**
* Implementation of the flow rule programmable behaviour for P4Runtime.
*/
-public class P4RuntimeFlowRuleProgrammable extends AbstractP4RuntimeHandlerBehaviour implements FlowRuleProgrammable {
+public class P4RuntimeFlowRuleProgrammable
+ extends AbstractP4RuntimeHandlerBehaviour
+ implements FlowRuleProgrammable {
- /*
- When updating an existing rule, if true, we issue a DELETE operation before inserting the new one, otherwise we
- issue a MODIFY operation. This is useful fore devices that do not support MODIFY operations for table entries.
- */
+ // When updating an existing rule, if true, we issue a DELETE operation
+ // before inserting the new one, otherwise we issue a MODIFY operation. This
+ // is useful fore devices that do not support MODIFY operations for table
+ // entries.
// TODO: make this attribute configurable by child drivers (e.g. BMv2 or Tofino)
private boolean deleteEntryBeforeUpdate = true;
- /*
- If true, we ignore re-installing rules that are already known in the ENTRY_STORE, i.e. same match key and action.
- */
- // TODO: can remove this check as soon as the multi-apply-per-same-flow rule bug is fixed.
- private boolean checkEntryStoreBeforeUpdate = true;
+ // If true, we ignore re-installing rules that are already exists the
+ // device, i.e. same match key and action.
+ // FIXME: can remove this check as soon as the multi-apply-per-same-flow rule bug is fixed.
+ private boolean checkStoreBeforeUpdate = true;
- /*
- If true, we avoid querying the device and return the content of the ENTRY_STORE.
- */
- // TODO: set to false after bmv2/PI bug fixed
+ // If true, we avoid querying the device and return what's already known by
+ // the ONOS store.
private boolean ignoreDeviceWhenGet = true;
- /*
- If true, we read all direct counters of a table with one request. Otherwise, send as many request as the number of
- table entries.
- */
- // TODO: set to true as soon as the feature is implemented in P4Runtime.
+ /* If true, we read all direct counters of a table with one request.
+ Otherwise, we send as many requests as the number of table entries. */
+ // FIXME: set to true as soon as the feature is implemented in P4Runtime.
private boolean readAllDirectCounters = false;
// Needed to synchronize operations over the same table entry.
- private static final ConcurrentMap<P4RuntimeTableEntryReference, Lock> ENTRY_LOCKS = Maps.newConcurrentMap();
-
- // TODO: replace with distributed store.
- // Can reuse old BMv2TableEntryService from ONOS 1.6
- private static final ConcurrentMap<P4RuntimeTableEntryReference, P4RuntimeFlowRuleWrapper> ENTRY_STORE =
- Maps.newConcurrentMap();
+ // FIXME: locks should be removed when unused (hint use cache with timeout)
+ private static final ConcurrentMap<PiTableEntryHandle, Lock>
+ ENTRY_LOCKS = Maps.newConcurrentMap();
private PiPipelineModel pipelineModel;
private PiPipelineInterpreter interpreter;
+ private P4RuntimeTableMirror tableMirror;
+ private PiFlowRuleTranslator translator;
@Override
protected boolean setupBehaviour() {
@@ -111,6 +113,8 @@
}
interpreter = device.as(PiPipelineInterpreter.class);
pipelineModel = pipeconf.pipelineModel();
+ tableMirror = handler().get(P4RuntimeTableMirror.class);
+ translator = piTranslationService.flowRuleTranslator();
return true;
}
@@ -122,98 +126,70 @@
}
if (ignoreDeviceWhenGet) {
- return ENTRY_STORE.values().stream()
- .filter(frWrapper -> frWrapper.rule().deviceId().equals(this.deviceId))
- .map(frWrapper -> new DefaultFlowEntry(frWrapper.rule(), ADDED, frWrapper.lifeInSeconds(),
- 0, 0))
- .collect(Collectors.toList());
+ return getFlowEntriesFromMirror();
}
- ImmutableList.Builder<FlowEntry> resultBuilder = ImmutableList.builder();
- List<PiTableEntry> inconsistentEntries = Lists.newArrayList();
+ final ImmutableList.Builder<FlowEntry> result = ImmutableList.builder();
+ final List<PiTableEntry> inconsistentEntries = Lists.newArrayList();
for (PiTableModel tableModel : pipelineModel.tables()) {
- PiTableId piTableId = tableModel.id();
+ final PiTableId piTableId = tableModel.id();
- Collection<PiTableEntry> installedEntries;
+ // Read table entries.
+ final Collection<PiTableEntry> installedEntries;
try {
- // TODO: optimize by dumping entries and counters in parallel, from ALL tables with the same request.
+ // TODO: optimize by dumping entries and counters in parallel
+ // From ALL tables with the same request.
installedEntries = client.dumpTable(piTableId, pipeconf).get();
} catch (InterruptedException | ExecutionException e) {
if (!(e.getCause() instanceof StatusRuntimeException)) {
// gRPC errors are logged in the client.
- log.error("Exception while dumping table {} of {}", piTableId, deviceId, e);
+ log.error("Exception while dumping table {} of {}",
+ piTableId, deviceId, e);
}
- return Collections.emptyList();
+ continue; // next table
}
- Map<PiTableEntry, PiCounterCellData> counterCellMap;
- try {
- if (interpreter.mapTableCounter(piTableId).isPresent()) {
- PiCounterId piCounterId = interpreter.mapTableCounter(piTableId).get();
- Collection<PiCounterCellData> cellDatas;
- if (readAllDirectCounters) {
- cellDatas = client.readAllCounterCells(Collections.singleton(piCounterId), pipeconf).get();
- } else {
- Set<PiCounterCellId> cellIds = installedEntries.stream()
- .map(entry -> PiCounterCellId.ofDirect(piCounterId, entry))
- .collect(Collectors.toSet());
- cellDatas = client.readCounterCells(cellIds, pipeconf).get();
- }
- counterCellMap = cellDatas.stream()
- .collect(Collectors.toMap(c -> (c.cellId()).tableEntry(), c -> c));
- } else {
- counterCellMap = Collections.emptyMap();
- }
- installedEntries = client.dumpTable(piTableId, pipeconf).get();
- } catch (InterruptedException | ExecutionException e) {
- if (!(e.getCause() instanceof StatusRuntimeException)) {
- // gRPC errors are logged in the client.
- log.error("Exception while reading counters of table {} of {}", piTableId, deviceId, e);
- }
+ if (installedEntries.size() == 0) {
+ continue; // next table
+ }
+
+ // Read table direct counters (if any).
+ final Map<PiTableEntry, PiCounterCellData> counterCellMap;
+ if (interpreter.mapTableCounter(piTableId).isPresent()) {
+ PiCounterId piCounterId = interpreter.mapTableCounter(piTableId).get();
+ counterCellMap = readEntryCounters(piCounterId, installedEntries);
+ } else {
counterCellMap = Collections.emptyMap();
}
+ // Forge flow entries with counter values.
for (PiTableEntry installedEntry : installedEntries) {
- P4RuntimeTableEntryReference entryRef = new P4RuntimeTableEntryReference(deviceId,
- piTableId,
- installedEntry.matchKey());
+ final FlowEntry flowEntry = forgeFlowEntry(
+ installedEntry, counterCellMap.get(installedEntry));
- if (!ENTRY_STORE.containsKey(entryRef)) {
- // Inconsistent entry
+ if (flowEntry == null) {
+ // Entry is on device but unknown to translation service or
+ // device mirror. Inconsistent. Mark for removal.
+ // TODO: make this behaviour configurable
+ // In some cases it's fine for the device to have rules
+ // that were not installed by us.
inconsistentEntries.add(installedEntry);
- continue; // next one.
+ } else {
+ result.add(flowEntry);
}
-
- P4RuntimeFlowRuleWrapper frWrapper = ENTRY_STORE.get(entryRef);
-
- long bytes = 0L;
- long packets = 0L;
- if (counterCellMap.containsKey(installedEntry)) {
- PiCounterCellData counterCellData = counterCellMap.get(installedEntry);
- bytes = counterCellData.bytes();
- packets = counterCellData.packets();
- }
-
- resultBuilder.add(new DefaultFlowEntry(frWrapper.rule(),
- ADDED,
- frWrapper.lifeInSeconds(),
- packets,
- bytes));
}
}
if (inconsistentEntries.size() > 0) {
- log.warn("Found {} entries in {} that are not known by table entry service," +
- " removing them", inconsistentEntries.size(), deviceId);
- inconsistentEntries.forEach(entry -> log.debug(entry.toString()));
- // Async remove them.
- client.writeTableEntries(inconsistentEntries, DELETE, pipeconf);
+ // Async clean up inconsistent entries.
+ SharedExecutors.getSingleThreadExecutor().execute(
+ () -> cleanUpInconsistentEntries(inconsistentEntries));
}
- return resultBuilder.build();
+ return result.build();
}
@Override
@@ -226,109 +202,206 @@
return processFlowRules(rules, REMOVE);
}
- private Collection<FlowRule> processFlowRules(Collection<FlowRule> rules, Operation operation) {
+ private FlowEntry forgeFlowEntry(PiTableEntry entry,
+ PiCounterCellData cellData) {
+ final PiTableEntryHandle handle = PiTableEntryHandle
+ .of(deviceId, entry);
+ final Optional<PiTranslatedEntity<FlowRule, PiTableEntry>>
+ translatedEntity = translator.lookup(handle);
+ final TimedEntry<PiTableEntry> timedEntry = tableMirror.get(handle);
+
+ if (!translatedEntity.isPresent()) {
+ log.debug("Handle not found in store: {}", handle);
+ return null;
+ }
+
+ if (timedEntry == null) {
+ log.debug("Handle not found in device mirror: {}", handle);
+ return null;
+ }
+
+ if (cellData != null) {
+ return new DefaultFlowEntry(translatedEntity.get().original(),
+ ADDED, timedEntry.lifeSec(), cellData.bytes(),
+ cellData.bytes());
+ } else {
+ return new DefaultFlowEntry(translatedEntity.get().original(),
+ ADDED, timedEntry.lifeSec(), 0, 0);
+ }
+ }
+
+ private Collection<FlowEntry> getFlowEntriesFromMirror() {
+ return tableMirror.getAll(deviceId).stream()
+ .map(timedEntry -> forgeFlowEntry(
+ timedEntry.entry(), null))
+ .collect(Collectors.toList());
+ }
+
+ private void cleanUpInconsistentEntries(Collection<PiTableEntry> piEntries) {
+ log.warn("Found {} entries from {} not on translation store, removing them...",
+ piEntries.size(), deviceId);
+ piEntries.forEach(entry -> {
+ log.debug(entry.toString());
+ applyEntry(PiTableEntryHandle.of(deviceId, entry),
+ entry, null, REMOVE);
+ });
+ }
+
+ private Collection<FlowRule> processFlowRules(Collection<FlowRule> rules,
+ Operation driverOperation) {
if (!setupBehaviour()) {
return Collections.emptyList();
}
- ImmutableList.Builder<FlowRule> processedFlowRuleListBuilder = ImmutableList.builder();
+ final ImmutableList.Builder<FlowRule> result = ImmutableList.builder();
- // TODO: send write operations in bulk (e.g. all entries to insert, modify or delete).
+ // TODO: send writes in bulk (e.g. all entries to insert, modify or delete).
// Instead of calling the client for each one of them.
- for (FlowRule rule : rules) {
+ for (FlowRule ruleToApply : rules) {
- PiTableEntry piTableEntry;
-
+ final PiTableEntry piEntryToApply;
try {
- piTableEntry = piTranslationService.translateFlowRule(rule, pipeconf);
- } catch (PiTranslationService.PiTranslationException e) {
- log.warn("Unable to translate flow rule: {} - {}", e.getMessage(), rule);
- continue; // next rule
+ piEntryToApply = translator.translate(ruleToApply, pipeconf);
+ } catch (PiTranslationException e) {
+ log.warn("Unable to translate flow rule for pipeconf '{}': {} - {}",
+ pipeconf.id(), e.getMessage(), ruleToApply);
+ // Next rule.
+ continue;
}
- PiTableId tableId = piTableEntry.table();
- P4RuntimeTableEntryReference entryRef = new P4RuntimeTableEntryReference(deviceId,
- tableId, piTableEntry.matchKey());
+ final PiTableEntryHandle handle = PiTableEntryHandle
+ .of(deviceId, piEntryToApply);
- Lock lock = ENTRY_LOCKS.computeIfAbsent(entryRef, k -> new ReentrantLock());
+ // Serialize operations over the same match key/table/device ID.
+ final Lock lock = ENTRY_LOCKS.computeIfAbsent(handle, k -> new ReentrantLock());
lock.lock();
-
try {
-
- P4RuntimeFlowRuleWrapper frWrapper = ENTRY_STORE.get(entryRef);
- WriteOperationType opType = null;
- boolean doApply = true;
-
- if (operation == APPLY) {
- if (frWrapper == null) {
- // Entry is first-timer.
- opType = INSERT;
- } else {
- // This match key already exists in the device.
- if (checkEntryStoreBeforeUpdate &&
- piTableEntry.action().equals(frWrapper.piTableEntry().action())) {
- doApply = false;
- log.debug("Ignoring re-apply of existing entry: {}", piTableEntry);
- }
- if (doApply) {
- if (deleteEntryBeforeUpdate) {
- // We've seen some strange error when trying to modify existing flow rules.
- // Remove before re-adding the modified one.
- try {
- if (client.writeTableEntries(newArrayList(piTableEntry), DELETE, pipeconf).get()) {
- frWrapper = null;
- } else {
- log.warn("Unable to DELETE table entry (before re-adding) in {}: {}",
- deviceId, piTableEntry);
- }
- } catch (InterruptedException | ExecutionException e) {
- log.warn("Exception while deleting table entry:", operation.name(), e);
- }
- opType = INSERT;
- } else {
- opType = MODIFY;
- }
- }
- }
- } else {
- opType = DELETE;
+ if (applyEntry(handle, piEntryToApply,
+ ruleToApply, driverOperation)) {
+ result.add(ruleToApply);
}
-
- if (doApply) {
- try {
- if (client.writeTableEntries(newArrayList(piTableEntry), opType, pipeconf).get()) {
- processedFlowRuleListBuilder.add(rule);
- if (operation == APPLY) {
- frWrapper = new P4RuntimeFlowRuleWrapper(rule, piTableEntry,
- System.currentTimeMillis());
- } else {
- frWrapper = null;
- }
- } else {
- log.warn("Unable to {} table entry in {}: {}", opType.name(), deviceId, piTableEntry);
- }
- } catch (InterruptedException | ExecutionException e) {
- log.warn("Exception while performing {} table entry operation:", operation.name(), e);
- }
- } else {
- processedFlowRuleListBuilder.add(rule);
- }
-
- // Update entryRef binding in table entry service.
- if (frWrapper != null) {
- ENTRY_STORE.put(entryRef, frWrapper);
- } else {
- ENTRY_STORE.remove(entryRef);
- }
-
} finally {
lock.unlock();
}
}
- return processedFlowRuleListBuilder.build();
+ return result.build();
+ }
+
+ /**
+ * Applies the given entry to the device, and returns true if the operation
+ * was successful, false otherwise.
+ */
+ private boolean applyEntry(PiTableEntryHandle handle,
+ PiTableEntry piEntryToApply,
+ FlowRule ruleToApply,
+ Operation driverOperation) {
+ // Depending on the driver operation, and if a matching rule exists on
+ // the device, decide which P4 Runtime write operation to perform for
+ // this entry.
+ final TimedEntry<PiTableEntry> piEntryOnDevice = tableMirror.get(handle);
+ final WriteOperationType p4Operation;
+ if (driverOperation == APPLY) {
+ if (piEntryOnDevice == null) {
+ // Entry is first-timer.
+ p4Operation = INSERT;
+ } else {
+ if (checkStoreBeforeUpdate
+ && piEntryToApply.action().equals(piEntryOnDevice.entry().action())) {
+ log.debug("Ignoring re-apply of existing entry: {}", piEntryToApply);
+ p4Operation = null;
+ } else if (deleteEntryBeforeUpdate) {
+ // Some devices return error when updating existing
+ // entries. If requested, remove entry before
+ // re-inserting the modified one.
+ applyEntry(handle, piEntryOnDevice.entry(), null, REMOVE);
+ p4Operation = INSERT;
+ } else {
+ p4Operation = MODIFY;
+ }
+ }
+ } else {
+ p4Operation = DELETE;
+ }
+
+ if (p4Operation != null) {
+ if (writeEntry(piEntryToApply, p4Operation)) {
+ updateStores(handle, piEntryToApply, ruleToApply, p4Operation);
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ // If no operation, let's pretend we applied the rule to the device.
+ return true;
+ }
+ }
+
+ /**
+ * Performs a write operation on the device.
+ */
+ private boolean writeEntry(PiTableEntry entry,
+ WriteOperationType p4Operation) {
+ try {
+ if (client.writeTableEntries(
+ newArrayList(entry), p4Operation, pipeconf).get()) {
+ return true;
+ } else {
+ log.warn("Unable to {} table entry in {}: {}",
+ p4Operation.name(), deviceId, entry);
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ log.warn("Exception while performing {} table entry operation:",
+ p4Operation, e);
+ }
+ return false;
+ }
+
+ private void updateStores(PiTableEntryHandle handle,
+ PiTableEntry entry,
+ FlowRule rule,
+ WriteOperationType p4Operation) {
+ switch (p4Operation) {
+ case INSERT:
+ case MODIFY:
+ tableMirror.put(handle, entry);
+ translator.learn(handle, new PiTranslatedEntity<>(rule, entry, handle));
+ break;
+ case DELETE:
+ tableMirror.remove(handle);
+ translator.forget(handle);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unknown operation " + p4Operation.name());
+ }
+ }
+
+ private Map<PiTableEntry, PiCounterCellData> readEntryCounters(
+ PiCounterId counterId, Collection<PiTableEntry> tableEntries) {
+ Collection<PiCounterCellData> cellDatas;
+ try {
+ if (readAllDirectCounters) {
+ cellDatas = client.readAllCounterCells(
+ singleton(counterId), pipeconf).get();
+ } else {
+ Set<PiCounterCellId> cellIds = tableEntries.stream()
+ .map(entry -> PiCounterCellId.ofDirect(counterId, entry))
+ .collect(Collectors.toSet());
+ cellDatas = client.readCounterCells(cellIds, pipeconf).get();
+ }
+ return cellDatas.stream()
+ .collect(Collectors.toMap(c -> c.cellId().tableEntry(), c -> c));
+ } catch (InterruptedException | ExecutionException e) {
+ if (!(e.getCause() instanceof StatusRuntimeException)) {
+ // gRPC errors are logged in the client.
+ log.error("Exception while reading counter '{}' from {}: {}",
+ counterId, deviceId, e);
+ }
+ return Collections.emptyMap();
+ }
}
enum Operation {
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java
index 464c4b6..fb5e892 100644
--- a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java
@@ -17,180 +17,86 @@
package org.onosproject.drivers.p4runtime;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.onosproject.core.GroupId;
-import org.onosproject.net.Device;
+import org.onosproject.drivers.p4runtime.mirror.P4RuntimeGroupMirror;
import org.onosproject.net.DeviceId;
-import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.group.DefaultGroup;
import org.onosproject.net.group.Group;
import org.onosproject.net.group.GroupOperation;
import org.onosproject.net.group.GroupOperations;
import org.onosproject.net.group.GroupProgrammable;
import org.onosproject.net.group.GroupStore;
import org.onosproject.net.pi.model.PiActionProfileId;
+import org.onosproject.net.pi.model.PiActionProfileModel;
import org.onosproject.net.pi.runtime.PiActionGroup;
-import org.onosproject.net.pi.runtime.PiActionGroupId;
-import org.onosproject.net.pi.service.PiTranslationService;
-import org.onosproject.p4runtime.api.P4RuntimeClient;
-import org.onosproject.p4runtime.api.P4RuntimeGroupReference;
-import org.onosproject.p4runtime.api.P4RuntimeGroupWrapper;
+import org.onosproject.net.pi.runtime.PiActionGroupHandle;
+import org.onosproject.net.pi.service.PiGroupTranslator;
+import org.onosproject.net.pi.service.PiTranslatedEntity;
+import org.onosproject.net.pi.service.PiTranslationException;
import org.slf4j.Logger;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.DELETE;
+import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.INSERT;
import static org.slf4j.LoggerFactory.getLogger;
/**
* Implementation of the group programmable behaviour for P4Runtime.
*/
-public class P4RuntimeGroupProgrammable extends AbstractP4RuntimeHandlerBehaviour implements GroupProgrammable {
- private static final String ACT_GRP_MEMS = "action group members";
- private static final String DELETE = "delete";
- private static final String ACT_GRP = "action group";
- private static final String INSERT = "insert";
+public class P4RuntimeGroupProgrammable
+ extends AbstractP4RuntimeHandlerBehaviour
+ implements GroupProgrammable {
+
+ private enum Operation {
+ APPLY, REMOVE
+ }
+
+ private static final String ACT_GRP_MEMS_STR = "action group members";
+ private static final String DELETE_STR = "delete";
+ private static final String ACT_GRP_STR = "action group";
+ private static final String INSERT_STR = "insert";
+
private static final Logger log = getLogger(P4RuntimeGroupProgrammable.class);
- /*
- * About action groups in P4runtime:
- * The type field is a place holder in p4runtime.proto right now, and we haven't defined it yet. You can assume all
- * the groups are "select" as per the OF spec. As a remainder, in the P4 terminology a member corresponds to an OF
- * bucket. Each member can also be used directly in the match table (kind of like an OF indirect group).
- */
+ // If true, we ignore re-installing groups that are already known in the
+ // device mirror.
+ private boolean checkMirrorBeforeUpdate = true;
- // TODO: make this attribute configurable by child drivers (e.g. BMv2 or Tofino)
- /*
- When updating an existing rule, if true, we issue a DELETE operation before inserting the new one, otherwise we
- issue a MODIFY operation. This is useful fore devices that do not support MODIFY operations for table entries.
- */
- private boolean deleteBeforeUpdate = true;
-
- // TODO: can remove this check as soon as the multi-apply-per-same-flow rule bug is fixed.
- /*
- If true, we ignore re-installing rules that are already known in the ENTRY_STORE, i.e. same match key and action.
- */
- private boolean checkStoreBeforeUpdate = true;
+ private GroupStore groupStore;
+ private P4RuntimeGroupMirror groupMirror;
+ private PiGroupTranslator translator;
// Needed to synchronize operations over the same group.
- private static final Map<P4RuntimeGroupReference, Lock> GROUP_LOCKS = Maps.newConcurrentMap();
-
- // TODO: replace with distribute store
- private static final Map<P4RuntimeGroupReference, P4RuntimeGroupWrapper> GROUP_STORE = Maps.newConcurrentMap();
+ private static final Map<PiActionGroupHandle, Lock> GROUP_LOCKS =
+ Maps.newConcurrentMap();
@Override
- public void performGroupOperation(DeviceId deviceId, GroupOperations groupOps) {
+ protected boolean setupBehaviour() {
+ if (!super.setupBehaviour()) {
+ return false;
+ }
+ groupMirror = this.handler().get(P4RuntimeGroupMirror.class);
+ groupStore = handler().get(GroupStore.class);
+ translator = piTranslationService.groupTranslator();
+ return true;
+ }
+
+ @Override
+ public void performGroupOperation(DeviceId deviceId,
+ GroupOperations groupOps) {
if (!setupBehaviour()) {
return;
}
-
- Device device = handler().get(DeviceService.class).getDevice(deviceId);
-
- for (GroupOperation groupOp : groupOps.operations()) {
- processGroupOp(device, groupOp);
- }
- }
-
- private void processGroupOp(Device device, GroupOperation groupOp) {
- GroupId groupId = groupOp.groupId();
- GroupStore groupStore = handler().get(GroupStore.class);
- Group group = groupStore.getGroup(device.id(), groupId);
-
- PiActionGroup piActionGroup;
- try {
- piActionGroup = piTranslationService.translateGroup(group, pipeconf);
- } catch (PiTranslationService.PiTranslationException e) {
- log.warn("Unable translate group, aborting group operation {}: {}", groupOp.opType(), e.getMessage());
- return;
- }
-
- P4RuntimeGroupReference groupRef = new P4RuntimeGroupReference(deviceId, piActionGroup.actionProfileId(),
- piActionGroup.id());
-
- Lock lock = GROUP_LOCKS.computeIfAbsent(groupRef, k -> new ReentrantLock());
- lock.lock();
-
- try {
- P4RuntimeGroupWrapper oldGroupWrapper = GROUP_STORE.get(groupRef);
- P4RuntimeGroupWrapper newGroupWrapper = new P4RuntimeGroupWrapper(piActionGroup, group,
- System.currentTimeMillis());
- switch (groupOp.opType()) {
- case ADD:
- case MODIFY:
- if (writeGroupToDevice(oldGroupWrapper, piActionGroup)) {
- GROUP_STORE.put(groupRef, newGroupWrapper);
- }
- break;
- case DELETE:
- if (deleteGroupFromDevice(piActionGroup)) {
- GROUP_STORE.remove(groupRef);
- }
- break;
- default:
- log.warn("Group operation {} not supported", groupOp.opType());
- }
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Installs action group and members to device via client interface.
- *
- * @param oldGroupWrapper old group wrapper for the group; null if not exists
- * @param piActionGroup the action group to be installed
- * @return true if install success; false otherwise
- */
- private boolean writeGroupToDevice(P4RuntimeGroupWrapper oldGroupWrapper, PiActionGroup piActionGroup) {
- boolean success = true;
- CompletableFuture<Boolean> writeSuccess;
- if (checkStoreBeforeUpdate && oldGroupWrapper != null &&
- oldGroupWrapper.piActionGroup().equals(piActionGroup)) {
- // Action group already exists, ignore it
- return true;
- }
- if (deleteBeforeUpdate && oldGroupWrapper != null) {
- success = deleteGroupFromDevice(oldGroupWrapper.piActionGroup());
- }
- writeSuccess = client.writeActionGroupMembers(piActionGroup,
- P4RuntimeClient.WriteOperationType.INSERT,
- pipeconf);
- success = success && completeSuccess(writeSuccess, ACT_GRP_MEMS, INSERT);
-
- writeSuccess = client.writeActionGroup(piActionGroup,
- P4RuntimeClient.WriteOperationType.INSERT,
- pipeconf);
- success = success && completeSuccess(writeSuccess, ACT_GRP, INSERT);
- return success;
- }
-
- private boolean deleteGroupFromDevice(PiActionGroup piActionGroup) {
- boolean success;
- CompletableFuture<Boolean> writeSuccess;
- writeSuccess = client.writeActionGroup(piActionGroup,
- P4RuntimeClient.WriteOperationType.DELETE,
- pipeconf);
- success = completeSuccess(writeSuccess, ACT_GRP, DELETE);
- writeSuccess = client.writeActionGroupMembers(piActionGroup,
- P4RuntimeClient.WriteOperationType.DELETE,
- pipeconf);
- success = success && completeSuccess(writeSuccess, ACT_GRP_MEMS, DELETE);
- return success;
- }
-
- private boolean completeSuccess(CompletableFuture<Boolean> completableFuture,
- String topic, String action) {
- try {
- return completableFuture.get();
- } catch (InterruptedException | ExecutionException e) {
- log.warn("Can't {} {} due to {}", action, topic, e.getMessage());
- return false;
- }
+ groupOps.operations().forEach(op -> processGroupOp(deviceId, op));
}
@Override
@@ -198,58 +104,147 @@
if (!setupBehaviour()) {
return Collections.emptyList();
}
+ return pipeconf.pipelineModel().actionProfiles().stream()
+ .map(PiActionProfileModel::id)
+ .flatMap(this::streamGroupsFromDevice)
+ .collect(Collectors.toList());
+ }
- Collection<Group> result = Sets.newHashSet();
- Collection<PiActionProfileId> piActionProfileIds = Sets.newHashSet();
+ private void processGroupOp(DeviceId deviceId, GroupOperation groupOp) {
+ final Group pdGroup = groupStore.getGroup(deviceId, groupOp.groupId());
- // TODO: find better way to get all action profile ids. e.g. by providing them in the interpreter
- GROUP_STORE.forEach((groupRef, wrapper) -> piActionProfileIds.add(groupRef.actionProfileId()));
+ final PiActionGroup piGroup;
+ try {
+ piGroup = translator.translate(pdGroup, pipeconf);
+ } catch (PiTranslationException e) {
+ log.warn("Unable translate group, aborting {} operation: {}",
+ groupOp.opType(), e.getMessage());
+ return;
+ }
- AtomicBoolean success = new AtomicBoolean(true);
- piActionProfileIds.forEach(actionProfileId -> {
- Collection<PiActionGroup> piActionGroups = Sets.newHashSet();
- try {
- Collection<PiActionGroup> groupsFromDevice =
- client.dumpGroups(actionProfileId, pipeconf).get();
- if (groupsFromDevice == null) {
- // Got error
- success.set(false);
- } else {
- piActionGroups.addAll(groupsFromDevice);
- }
- } catch (ExecutionException | InterruptedException e) {
- log.error("Exception while dumping groups for action profile {}: {}",
- actionProfileId.id(), deviceId, e);
- success.set(false);
+ final PiActionGroupHandle handle = PiActionGroupHandle.of(deviceId, piGroup);
+
+ final PiActionGroup groupOnDevice = groupMirror.get(handle) == null
+ ? null
+ : groupMirror.get(handle).entry();
+
+ final Lock lock = GROUP_LOCKS.computeIfAbsent(handle, k -> new ReentrantLock());
+ lock.lock();
+ try {
+ final Operation operation;
+ switch (groupOp.opType()) {
+ case ADD:
+ case MODIFY:
+ operation = Operation.APPLY;
+ break;
+ case DELETE:
+ operation = Operation.REMOVE;
+ break;
+ default:
+ log.warn("Group operation {} not supported", groupOp.opType());
+ return;
}
-
- piActionGroups.forEach(piActionGroup -> {
- PiActionGroupId actionGroupId = piActionGroup.id();
- P4RuntimeGroupReference groupRef =
- new P4RuntimeGroupReference(deviceId, actionProfileId, actionGroupId);
- P4RuntimeGroupWrapper wrapper = GROUP_STORE.get(groupRef);
-
- if (wrapper == null) {
- // group exists in client, but can't find in ONOS
- log.warn("Can't find action profile group {} from local store.",
- groupRef);
- return;
- }
- if (!wrapper.piActionGroup().equals(piActionGroup)) {
- log.warn("Group from device is different to group from local store.");
- return;
- }
- result.add(wrapper.group());
-
- });
- });
-
- if (!success.get()) {
- // Got error while dump groups from device.
- return Collections.emptySet();
- } else {
- return result;
+ processPiGroup(handle, piGroup,
+ groupOnDevice, pdGroup, operation);
+ } finally {
+ lock.unlock();
}
}
+ private void processPiGroup(PiActionGroupHandle handle,
+ PiActionGroup groupToApply,
+ PiActionGroup groupOnDevice,
+ Group pdGroup, Operation operation) {
+ if (operation == Operation.APPLY) {
+ if (groupOnDevice != null) {
+ if (checkMirrorBeforeUpdate
+ && groupOnDevice.equals(groupToApply)) {
+ // Group on device has the same members, ignore operation.
+ return;
+ }
+ // Remove before adding it.
+ processPiGroup(handle, groupToApply, groupOnDevice,
+ pdGroup, Operation.REMOVE);
+ }
+ if (writeGroupToDevice(groupToApply)) {
+ groupMirror.put(handle, groupToApply);
+ translator.learn(handle, new PiTranslatedEntity<>(
+ pdGroup, groupToApply, handle));
+ }
+ } else {
+ if (deleteGroupFromDevice(groupToApply)) {
+ groupMirror.remove(handle);
+ translator.forget(handle);
+ }
+ }
+ }
+
+ private boolean writeGroupToDevice(PiActionGroup groupToApply) {
+ // First insert members, then group.
+ // The operation is deemed successful if both operations are successful.
+ // FIXME: add transactional semantics, i.e. remove members if group fails.
+ final boolean membersSuccess = completeFuture(
+ client.writeActionGroupMembers(groupToApply, INSERT, pipeconf),
+ ACT_GRP_MEMS_STR, INSERT_STR);
+ return membersSuccess && completeFuture(
+ client.writeActionGroup(groupToApply, INSERT, pipeconf),
+ ACT_GRP_STR, INSERT_STR);
+ }
+
+ private boolean deleteGroupFromDevice(PiActionGroup piActionGroup) {
+ // First delete group, then members.
+ // The operation is deemed successful if both operations are successful.
+ final boolean groupSuccess = completeFuture(
+ client.writeActionGroup(piActionGroup, DELETE, pipeconf),
+ ACT_GRP_STR, DELETE_STR);
+ return groupSuccess && completeFuture(
+ client.writeActionGroupMembers(piActionGroup, DELETE, pipeconf),
+ ACT_GRP_MEMS_STR, DELETE_STR);
+ }
+
+ private boolean completeFuture(CompletableFuture<Boolean> completableFuture,
+ String topic, String action) {
+ try {
+ if (completableFuture.get()) {
+ return true;
+ } else {
+ log.warn("Unable to {} {}", action, topic);
+ return false;
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ log.warn("Exception while performing {} {}: {}", action, topic, e.getMessage());
+ log.debug("Exception", e);
+ return false;
+ }
+ }
+
+ private Stream<Group> streamGroupsFromDevice(PiActionProfileId actProfId) {
+ try {
+ // Read PI groups and return original PD one.
+ return client.dumpGroups(actProfId, pipeconf).get().stream()
+ .map(this::forgeGroupEntry)
+ .filter(Objects::nonNull);
+ } catch (ExecutionException | InterruptedException e) {
+ log.error("Exception while dumping groups from action profile '{}' on {}: {}",
+ actProfId.id(), deviceId, e);
+ return Stream.empty();
+ }
+ }
+
+ private Group forgeGroupEntry(PiActionGroup piGroup) {
+ final PiActionGroupHandle handle = PiActionGroupHandle.of(deviceId, piGroup);
+ if (!translator.lookup(handle).isPresent()) {
+ log.warn("Missing PI group from translation store: {} - {}:{}",
+ pipeconf.id(), piGroup.actionProfileId(),
+ piGroup.id());
+ return null;
+ }
+ final long life = groupMirror.get(handle) != null
+ ? groupMirror.get(handle).lifeSec() : 0;
+ final Group original = translator.lookup(handle).get().original();
+ final DefaultGroup forgedGroup = new DefaultGroup(original.id(), original);
+ forgedGroup.setState(Group.GroupState.ADDED);
+ forgedGroup.setLife(life);
+ return forgedGroup;
+ }
}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeHandshaker.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeHandshaker.java
index 7e41a99..f49ad18 100644
--- a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeHandshaker.java
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeHandshaker.java
@@ -66,7 +66,7 @@
client = controller.getClient(deviceId);
if (client == null || !controller.isReacheable(deviceId)) {
- result.complete(MastershipRole.STANDBY);
+ result.complete(MastershipRole.NONE);
return result;
}
if (newRole.equals(MastershipRole.MASTER)) {
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/AbstractDistributedP4RuntimeMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/AbstractDistributedP4RuntimeMirror.java
new file mode 100644
index 0000000..be0b0b3
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/AbstractDistributedP4RuntimeMirror.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import com.google.common.annotations.Beta;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.pi.runtime.PiEntity;
+import org.onosproject.net.pi.runtime.PiHandle;
+import org.onosproject.store.service.EventuallyConsistentMap;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.WallClockTimestamp;
+import org.slf4j.Logger;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Abstract implementation of a distributed P4Runtime mirror, backed by an
+ * {@link EventuallyConsistentMap}.
+ *
+ * @param <H> handle class
+ * @param <E> entry class
+ */
+@Beta
+@Component(immediate = true)
+public abstract class AbstractDistributedP4RuntimeMirror
+ <H extends PiHandle, E extends PiEntity>
+ implements P4RuntimeMirror<H, E> {
+
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ private StorageService storageService;
+
+ private EventuallyConsistentMap<H, TimedEntry<E>> mirrorMap;
+
+ @Activate
+ public void activate() {
+ mirrorMap = storageService
+ .<H, TimedEntry<E>>eventuallyConsistentMapBuilder()
+ .withName(mapName())
+ .withSerializer(storeSerializer())
+ .withTimestampProvider((k, v) -> new WallClockTimestamp())
+ .build();
+ log.info("Started");
+ }
+
+ abstract String mapName();
+
+ abstract KryoNamespace storeSerializer();
+
+ @Deactivate
+ public void deactivate() {
+ mirrorMap = null;
+ log.info("Stopped");
+ }
+
+ @Override
+ public Collection<TimedEntry<E>> getAll(DeviceId deviceId) {
+ checkNotNull(deviceId);
+ return mirrorMap.entrySet().stream()
+ .filter(entry -> entry.getKey().deviceId().equals(deviceId))
+ .map(Map.Entry::getValue)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public TimedEntry<E> get(H handle) {
+ checkNotNull(handle);
+ return mirrorMap.get(handle);
+ }
+
+ @Override
+ public void put(H handle, E entry) {
+ checkNotNull(handle);
+ checkNotNull(entry);
+ final long now = new WallClockTimestamp().unixTimestamp();
+ final TimedEntry<E> timedEntry = new TimedEntry<>(now, entry);
+ mirrorMap.put(handle, timedEntry);
+ }
+
+ @Override
+ public void remove(H handle) {
+ checkNotNull(handle);
+ mirrorMap.remove(handle);
+ }
+
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeGroupMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeGroupMirror.java
new file mode 100644
index 0000000..4c963a6
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeGroupMirror.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.net.pi.runtime.PiActionGroup;
+import org.onosproject.net.pi.runtime.PiActionGroupHandle;
+import org.onosproject.store.serializers.KryoNamespaces;
+
+/**
+ * Distributed implementation of a P4Runtime group mirror.
+ */
+@Component(immediate = true)
+@Service
+public final class DistributedP4RuntimeGroupMirror
+ extends AbstractDistributedP4RuntimeMirror
+ <PiActionGroupHandle, PiActionGroup>
+ implements P4RuntimeGroupMirror {
+
+ private static final String DIST_MAP_NAME = "onos-p4runtime-group-mirror";
+
+ @Override
+ String mapName() {
+ return DIST_MAP_NAME;
+ }
+
+ @Override
+ KryoNamespace storeSerializer() {
+ return KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(TimedEntry.class)
+ .build();
+ }
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeTableMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeTableMirror.java
new file mode 100644
index 0000000..f37cf44
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeTableMirror.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.net.pi.runtime.PiTableEntry;
+import org.onosproject.net.pi.runtime.PiTableEntryHandle;
+import org.onosproject.store.serializers.KryoNamespaces;
+
+/**
+ * Distributed implementation of a P4Runtime table mirror.
+ */
+@Component(immediate = true)
+@Service
+public final class DistributedP4RuntimeTableMirror
+ extends AbstractDistributedP4RuntimeMirror
+ <PiTableEntryHandle, PiTableEntry>
+ implements P4RuntimeTableMirror {
+
+ private static final String DIST_MAP_NAME = "onos-p4runtime-table-mirror";
+
+ @Override
+ String mapName() {
+ return DIST_MAP_NAME;
+ }
+
+ @Override
+ KryoNamespace storeSerializer() {
+ return KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(TimedEntry.class)
+ .build();
+ }
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeGroupMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeGroupMirror.java
new file mode 100644
index 0000000..f363e71
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeGroupMirror.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import org.onosproject.net.pi.runtime.PiActionGroup;
+import org.onosproject.net.pi.runtime.PiActionGroupHandle;
+
+/**
+ * Mirror of action groups installed on a P4Runtime device.
+ */
+public interface P4RuntimeGroupMirror
+ extends P4RuntimeMirror<PiActionGroupHandle, PiActionGroup> {
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeMirror.java
new file mode 100644
index 0000000..ab18c9d
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeMirror.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import com.google.common.annotations.Beta;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.pi.runtime.PiEntity;
+import org.onosproject.net.pi.runtime.PiHandle;
+
+import java.util.Collection;
+
+/**
+ * Service to keep track of the device state for a given class of PI entities.
+ * The need of this service comes from the fact that P4 Runtime makes a
+ * distinction between INSERT and MODIFY operations, while ONOS drivers use a
+ * more generic "APPLY" behaviour (i.e. ADD or UPDATE). When applying an entry,
+ * we need to know if another one with the same handle (e.g. table entry with
+ * same match key) is already on the device to decide between INSERT or MODIFY.
+ * Moreover, this service maintains a "timed" version of PI entities such that
+ * we can compute the life of the entity on the device.
+ *
+ * @param <H> Handle class
+ * @param <E> Entity class
+ */
+@Beta
+public interface P4RuntimeMirror
+ <H extends PiHandle, E extends PiEntity> {
+
+ /**
+ * Returns all entries for the given device ID.
+ *
+ * @param deviceId device ID
+ * @return collection of table entries
+ */
+ Collection<TimedEntry<E>> getAll(DeviceId deviceId);
+
+ /**
+ * Returns entry associated to the given handle, if present, otherwise
+ * null.
+ *
+ * @param handle handle
+ * @return PI table entry
+ */
+ TimedEntry<E> get(H handle);
+
+ /**
+ * Stores the given entry associating it to the given handle.
+ *
+ * @param handle handle
+ * @param entry entry
+ */
+ void put(H handle, E entry);
+
+ /**
+ * Removes the entry associated to the given handle.
+ *
+ * @param handle handle
+ */
+ void remove(H handle);
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeTableMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeTableMirror.java
new file mode 100644
index 0000000..318e2b0
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeTableMirror.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import org.onosproject.net.pi.runtime.PiTableEntry;
+import org.onosproject.net.pi.runtime.PiTableEntryHandle;
+
+/**
+ * Mirror of table entries installed on a P4Runtime device.
+ */
+public interface P4RuntimeTableMirror
+ extends P4RuntimeMirror<PiTableEntryHandle, PiTableEntry> {
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/TimedEntry.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/TimedEntry.java
new file mode 100644
index 0000000..76b44a0
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/TimedEntry.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import org.onosproject.net.pi.runtime.PiEntity;
+import org.onosproject.store.service.WallClockTimestamp;
+
+public class TimedEntry<E extends PiEntity> {
+
+ private final long timestamp;
+ private final E entity;
+
+ TimedEntry(long timestamp, E entity) {
+ this.timestamp = timestamp;
+ this.entity = entity;
+ }
+
+ public long timestamp() {
+ return timestamp;
+ }
+
+ public E entry() {
+ return entity;
+ }
+
+ public long lifeSec() {
+ final long now = new WallClockTimestamp().unixTimestamp();
+ return (now - timestamp) / 1000;
+ }
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/package-info.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/package-info.java
new file mode 100644
index 0000000..d9b21d6
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * P4 Runtime device mirror.
+ */
+package org.onosproject.drivers.p4runtime.mirror;