ONOS-7050 Refactored P4Runtime FRP to use distributed stores
It uses the PI translation store and a newly introduced P4Runtime device
mirror.
Change-Id: Id2031af5e9bbdc8be4ec6967b867f97d35d54ab0
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
index 053dd7c..ca9392f 100644
--- a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
@@ -20,6 +20,9 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import io.grpc.StatusRuntimeException;
+import org.onlab.util.SharedExecutors;
+import org.onosproject.drivers.p4runtime.mirror.P4RuntimeTableMirror;
+import org.onosproject.drivers.p4runtime.mirror.TimedEntry;
import org.onosproject.net.flow.DefaultFlowEntry;
import org.onosproject.net.flow.FlowEntry;
import org.onosproject.net.flow.FlowRule;
@@ -32,15 +35,17 @@
import org.onosproject.net.pi.runtime.PiCounterCellData;
import org.onosproject.net.pi.runtime.PiCounterCellId;
import org.onosproject.net.pi.runtime.PiTableEntry;
+import org.onosproject.net.pi.runtime.PiTableEntryHandle;
+import org.onosproject.net.pi.service.PiFlowRuleTranslator;
+import org.onosproject.net.pi.service.PiTranslatedEntity;
import org.onosproject.net.pi.service.PiTranslationException;
import org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType;
-import org.onosproject.p4runtime.api.P4RuntimeFlowRuleWrapper;
-import org.onosproject.p4runtime.api.P4RuntimeTableEntryReference;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
@@ -49,6 +54,7 @@
import java.util.stream.Collectors;
import static com.google.common.collect.Lists.newArrayList;
+import static java.util.Collections.singleton;
import static org.onosproject.drivers.p4runtime.P4RuntimeFlowRuleProgrammable.Operation.APPLY;
import static org.onosproject.drivers.p4runtime.P4RuntimeFlowRuleProgrammable.Operation.REMOVE;
import static org.onosproject.net.flow.FlowEntry.FlowEntryState.ADDED;
@@ -59,44 +65,40 @@
/**
* Implementation of the flow rule programmable behaviour for P4Runtime.
*/
-public class P4RuntimeFlowRuleProgrammable extends AbstractP4RuntimeHandlerBehaviour implements FlowRuleProgrammable {
+public class P4RuntimeFlowRuleProgrammable
+ extends AbstractP4RuntimeHandlerBehaviour
+ implements FlowRuleProgrammable {
- /*
- When updating an existing rule, if true, we issue a DELETE operation before inserting the new one, otherwise we
- issue a MODIFY operation. This is useful fore devices that do not support MODIFY operations for table entries.
- */
+ // When updating an existing rule, if true, we issue a DELETE operation
+ // before inserting the new one, otherwise we issue a MODIFY operation. This
+ // is useful fore devices that do not support MODIFY operations for table
+ // entries.
// TODO: make this attribute configurable by child drivers (e.g. BMv2 or Tofino)
private boolean deleteEntryBeforeUpdate = true;
- /*
- If true, we ignore re-installing rules that are already known in the ENTRY_STORE, i.e. same match key and action.
- */
- // TODO: can remove this check as soon as the multi-apply-per-same-flow rule bug is fixed.
- private boolean checkEntryStoreBeforeUpdate = true;
+ // If true, we ignore re-installing rules that are already exists the
+ // device, i.e. same match key and action.
+ // FIXME: can remove this check as soon as the multi-apply-per-same-flow rule bug is fixed.
+ private boolean checkStoreBeforeUpdate = true;
- /*
- If true, we avoid querying the device and return the content of the ENTRY_STORE.
- */
- // TODO: set to false after bmv2/PI bug fixed
+ // If true, we avoid querying the device and return what's already known by
+ // the ONOS store.
private boolean ignoreDeviceWhenGet = true;
- /*
- If true, we read all direct counters of a table with one request. Otherwise, send as many request as the number of
- table entries.
- */
- // TODO: set to true as soon as the feature is implemented in P4Runtime.
+ /* If true, we read all direct counters of a table with one request.
+ Otherwise, we send as many requests as the number of table entries. */
+ // FIXME: set to true as soon as the feature is implemented in P4Runtime.
private boolean readAllDirectCounters = false;
// Needed to synchronize operations over the same table entry.
- private static final ConcurrentMap<P4RuntimeTableEntryReference, Lock> ENTRY_LOCKS = Maps.newConcurrentMap();
-
- // TODO: replace with distributed store.
- // Can reuse old BMv2TableEntryService from ONOS 1.6
- private static final ConcurrentMap<P4RuntimeTableEntryReference, P4RuntimeFlowRuleWrapper> ENTRY_STORE =
- Maps.newConcurrentMap();
+ // FIXME: locks should be removed when unused (hint use cache with timeout)
+ private static final ConcurrentMap<PiTableEntryHandle, Lock>
+ ENTRY_LOCKS = Maps.newConcurrentMap();
private PiPipelineModel pipelineModel;
private PiPipelineInterpreter interpreter;
+ private P4RuntimeTableMirror tableMirror;
+ private PiFlowRuleTranslator translator;
@Override
protected boolean setupBehaviour() {
@@ -111,6 +113,8 @@
}
interpreter = device.as(PiPipelineInterpreter.class);
pipelineModel = pipeconf.pipelineModel();
+ tableMirror = handler().get(P4RuntimeTableMirror.class);
+ translator = piTranslationService.flowRuleTranslator();
return true;
}
@@ -122,98 +126,70 @@
}
if (ignoreDeviceWhenGet) {
- return ENTRY_STORE.values().stream()
- .filter(frWrapper -> frWrapper.rule().deviceId().equals(this.deviceId))
- .map(frWrapper -> new DefaultFlowEntry(frWrapper.rule(), ADDED, frWrapper.lifeInSeconds(),
- 0, 0))
- .collect(Collectors.toList());
+ return getFlowEntriesFromMirror();
}
- ImmutableList.Builder<FlowEntry> resultBuilder = ImmutableList.builder();
- List<PiTableEntry> inconsistentEntries = Lists.newArrayList();
+ final ImmutableList.Builder<FlowEntry> result = ImmutableList.builder();
+ final List<PiTableEntry> inconsistentEntries = Lists.newArrayList();
for (PiTableModel tableModel : pipelineModel.tables()) {
- PiTableId piTableId = tableModel.id();
+ final PiTableId piTableId = tableModel.id();
- Collection<PiTableEntry> installedEntries;
+ // Read table entries.
+ final Collection<PiTableEntry> installedEntries;
try {
- // TODO: optimize by dumping entries and counters in parallel, from ALL tables with the same request.
+ // TODO: optimize by dumping entries and counters in parallel
+ // From ALL tables with the same request.
installedEntries = client.dumpTable(piTableId, pipeconf).get();
} catch (InterruptedException | ExecutionException e) {
if (!(e.getCause() instanceof StatusRuntimeException)) {
// gRPC errors are logged in the client.
- log.error("Exception while dumping table {} of {}", piTableId, deviceId, e);
+ log.error("Exception while dumping table {} of {}",
+ piTableId, deviceId, e);
}
- return Collections.emptyList();
+ continue; // next table
}
- Map<PiTableEntry, PiCounterCellData> counterCellMap;
- try {
- if (interpreter.mapTableCounter(piTableId).isPresent()) {
- PiCounterId piCounterId = interpreter.mapTableCounter(piTableId).get();
- Collection<PiCounterCellData> cellDatas;
- if (readAllDirectCounters) {
- cellDatas = client.readAllCounterCells(Collections.singleton(piCounterId), pipeconf).get();
- } else {
- Set<PiCounterCellId> cellIds = installedEntries.stream()
- .map(entry -> PiCounterCellId.ofDirect(piCounterId, entry))
- .collect(Collectors.toSet());
- cellDatas = client.readCounterCells(cellIds, pipeconf).get();
- }
- counterCellMap = cellDatas.stream()
- .collect(Collectors.toMap(c -> (c.cellId()).tableEntry(), c -> c));
- } else {
- counterCellMap = Collections.emptyMap();
- }
- installedEntries = client.dumpTable(piTableId, pipeconf).get();
- } catch (InterruptedException | ExecutionException e) {
- if (!(e.getCause() instanceof StatusRuntimeException)) {
- // gRPC errors are logged in the client.
- log.error("Exception while reading counters of table {} of {}", piTableId, deviceId, e);
- }
+ if (installedEntries.size() == 0) {
+ continue; // next table
+ }
+
+ // Read table direct counters (if any).
+ final Map<PiTableEntry, PiCounterCellData> counterCellMap;
+ if (interpreter.mapTableCounter(piTableId).isPresent()) {
+ PiCounterId piCounterId = interpreter.mapTableCounter(piTableId).get();
+ counterCellMap = readEntryCounters(piCounterId, installedEntries);
+ } else {
counterCellMap = Collections.emptyMap();
}
+ // Forge flow entries with counter values.
for (PiTableEntry installedEntry : installedEntries) {
- P4RuntimeTableEntryReference entryRef = new P4RuntimeTableEntryReference(deviceId,
- piTableId,
- installedEntry.matchKey());
+ final FlowEntry flowEntry = forgeFlowEntry(
+ installedEntry, counterCellMap.get(installedEntry));
- if (!ENTRY_STORE.containsKey(entryRef)) {
- // Inconsistent entry
+ if (flowEntry == null) {
+ // Entry is on device but unknown to translation service or
+ // device mirror. Inconsistent. Mark for removal.
+ // TODO: make this behaviour configurable
+ // In some cases it's fine for the device to have rules
+ // that were not installed by us.
inconsistentEntries.add(installedEntry);
- continue; // next one.
+ } else {
+ result.add(flowEntry);
}
-
- P4RuntimeFlowRuleWrapper frWrapper = ENTRY_STORE.get(entryRef);
-
- long bytes = 0L;
- long packets = 0L;
- if (counterCellMap.containsKey(installedEntry)) {
- PiCounterCellData counterCellData = counterCellMap.get(installedEntry);
- bytes = counterCellData.bytes();
- packets = counterCellData.packets();
- }
-
- resultBuilder.add(new DefaultFlowEntry(frWrapper.rule(),
- ADDED,
- frWrapper.lifeInSeconds(),
- packets,
- bytes));
}
}
if (inconsistentEntries.size() > 0) {
- log.warn("Found {} entries in {} that are not known by table entry service," +
- " removing them", inconsistentEntries.size(), deviceId);
- inconsistentEntries.forEach(entry -> log.debug(entry.toString()));
- // Async remove them.
- client.writeTableEntries(inconsistentEntries, DELETE, pipeconf);
+ // Async clean up inconsistent entries.
+ SharedExecutors.getSingleThreadExecutor().execute(
+ () -> cleanUpInconsistentEntries(inconsistentEntries));
}
- return resultBuilder.build();
+ return result.build();
}
@Override
@@ -226,109 +202,206 @@
return processFlowRules(rules, REMOVE);
}
- private Collection<FlowRule> processFlowRules(Collection<FlowRule> rules, Operation operation) {
+ private FlowEntry forgeFlowEntry(PiTableEntry entry,
+ PiCounterCellData cellData) {
+ final PiTableEntryHandle handle = PiTableEntryHandle
+ .of(deviceId, entry);
+ final Optional<PiTranslatedEntity<FlowRule, PiTableEntry>>
+ translatedEntity = translator.lookup(handle);
+ final TimedEntry<PiTableEntry> timedEntry = tableMirror.get(handle);
+
+ if (!translatedEntity.isPresent()) {
+ log.debug("Handle not found in store: {}", handle);
+ return null;
+ }
+
+ if (timedEntry == null) {
+ log.debug("Handle not found in device mirror: {}", handle);
+ return null;
+ }
+
+ if (cellData != null) {
+ return new DefaultFlowEntry(translatedEntity.get().original(),
+ ADDED, timedEntry.lifeSec(), cellData.bytes(),
+ cellData.bytes());
+ } else {
+ return new DefaultFlowEntry(translatedEntity.get().original(),
+ ADDED, timedEntry.lifeSec(), 0, 0);
+ }
+ }
+
+ private Collection<FlowEntry> getFlowEntriesFromMirror() {
+ return tableMirror.getAll(deviceId).stream()
+ .map(timedEntry -> forgeFlowEntry(
+ timedEntry.entry(), null))
+ .collect(Collectors.toList());
+ }
+
+ private void cleanUpInconsistentEntries(Collection<PiTableEntry> piEntries) {
+ log.warn("Found {} entries from {} not on translation store, removing them...",
+ piEntries.size(), deviceId);
+ piEntries.forEach(entry -> {
+ log.debug(entry.toString());
+ applyEntry(PiTableEntryHandle.of(deviceId, entry),
+ entry, null, REMOVE);
+ });
+ }
+
+ private Collection<FlowRule> processFlowRules(Collection<FlowRule> rules,
+ Operation driverOperation) {
if (!setupBehaviour()) {
return Collections.emptyList();
}
- ImmutableList.Builder<FlowRule> processedFlowRuleListBuilder = ImmutableList.builder();
+ final ImmutableList.Builder<FlowRule> result = ImmutableList.builder();
- // TODO: send write operations in bulk (e.g. all entries to insert, modify or delete).
+ // TODO: send writes in bulk (e.g. all entries to insert, modify or delete).
// Instead of calling the client for each one of them.
- for (FlowRule rule : rules) {
+ for (FlowRule ruleToApply : rules) {
- PiTableEntry piTableEntry;
-
+ final PiTableEntry piEntryToApply;
try {
- piTableEntry = piTranslationService.flowRuleTranslator().translate(rule, pipeconf);
+ piEntryToApply = translator.translate(ruleToApply, pipeconf);
} catch (PiTranslationException e) {
- log.warn("Unable to translate flow rule: {} - {}", e.getMessage(), rule);
- continue; // next rule
+ log.warn("Unable to translate flow rule for pipeconf '{}': {} - {}",
+ pipeconf.id(), e.getMessage(), ruleToApply);
+ // Next rule.
+ continue;
}
- PiTableId tableId = piTableEntry.table();
- P4RuntimeTableEntryReference entryRef = new P4RuntimeTableEntryReference(deviceId,
- tableId, piTableEntry.matchKey());
+ final PiTableEntryHandle handle = PiTableEntryHandle
+ .of(deviceId, piEntryToApply);
- Lock lock = ENTRY_LOCKS.computeIfAbsent(entryRef, k -> new ReentrantLock());
+ // Serialize operations over the same match key/table/device ID.
+ final Lock lock = ENTRY_LOCKS.computeIfAbsent(handle, k -> new ReentrantLock());
lock.lock();
-
try {
-
- P4RuntimeFlowRuleWrapper frWrapper = ENTRY_STORE.get(entryRef);
- WriteOperationType opType = null;
- boolean doApply = true;
-
- if (operation == APPLY) {
- if (frWrapper == null) {
- // Entry is first-timer.
- opType = INSERT;
- } else {
- // This match key already exists in the device.
- if (checkEntryStoreBeforeUpdate &&
- piTableEntry.action().equals(frWrapper.piTableEntry().action())) {
- doApply = false;
- log.debug("Ignoring re-apply of existing entry: {}", piTableEntry);
- }
- if (doApply) {
- if (deleteEntryBeforeUpdate) {
- // We've seen some strange error when trying to modify existing flow rules.
- // Remove before re-adding the modified one.
- try {
- if (client.writeTableEntries(newArrayList(piTableEntry), DELETE, pipeconf).get()) {
- frWrapper = null;
- } else {
- log.warn("Unable to DELETE table entry (before re-adding) in {}: {}",
- deviceId, piTableEntry);
- }
- } catch (InterruptedException | ExecutionException e) {
- log.warn("Exception while deleting table entry:", operation.name(), e);
- }
- opType = INSERT;
- } else {
- opType = MODIFY;
- }
- }
- }
- } else {
- opType = DELETE;
+ if (applyEntry(handle, piEntryToApply,
+ ruleToApply, driverOperation)) {
+ result.add(ruleToApply);
}
-
- if (doApply) {
- try {
- if (client.writeTableEntries(newArrayList(piTableEntry), opType, pipeconf).get()) {
- processedFlowRuleListBuilder.add(rule);
- if (operation == APPLY) {
- frWrapper = new P4RuntimeFlowRuleWrapper(rule, piTableEntry,
- System.currentTimeMillis());
- } else {
- frWrapper = null;
- }
- } else {
- log.warn("Unable to {} table entry in {}: {}", opType.name(), deviceId, piTableEntry);
- }
- } catch (InterruptedException | ExecutionException e) {
- log.warn("Exception while performing {} table entry operation:", operation.name(), e);
- }
- } else {
- processedFlowRuleListBuilder.add(rule);
- }
-
- // Update entryRef binding in table entry service.
- if (frWrapper != null) {
- ENTRY_STORE.put(entryRef, frWrapper);
- } else {
- ENTRY_STORE.remove(entryRef);
- }
-
} finally {
lock.unlock();
}
}
- return processedFlowRuleListBuilder.build();
+ return result.build();
+ }
+
+ /**
+ * Applies the given entry to the device, and returns true if the operation
+ * was successful, false otherwise.
+ */
+ private boolean applyEntry(PiTableEntryHandle handle,
+ PiTableEntry piEntryToApply,
+ FlowRule ruleToApply,
+ Operation driverOperation) {
+ // Depending on the driver operation, and if a matching rule exists on
+ // the device, decide which P4 Runtime write operation to perform for
+ // this entry.
+ final TimedEntry<PiTableEntry> piEntryOnDevice = tableMirror.get(handle);
+ final WriteOperationType p4Operation;
+ if (driverOperation == APPLY) {
+ if (piEntryOnDevice == null) {
+ // Entry is first-timer.
+ p4Operation = INSERT;
+ } else {
+ if (checkStoreBeforeUpdate
+ && piEntryToApply.action().equals(piEntryOnDevice.entry().action())) {
+ log.debug("Ignoring re-apply of existing entry: {}", piEntryToApply);
+ p4Operation = null;
+ } else if (deleteEntryBeforeUpdate) {
+ // Some devices return error when updating existing
+ // entries. If requested, remove entry before
+ // re-inserting the modified one.
+ applyEntry(handle, piEntryOnDevice.entry(), null, REMOVE);
+ p4Operation = INSERT;
+ } else {
+ p4Operation = MODIFY;
+ }
+ }
+ } else {
+ p4Operation = DELETE;
+ }
+
+ if (p4Operation != null) {
+ if (writeEntry(piEntryToApply, p4Operation)) {
+ updateStores(handle, piEntryToApply, ruleToApply, p4Operation);
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ // If no operation, let's pretend we applied the rule to the device.
+ return true;
+ }
+ }
+
+ /**
+ * Performs a write operation on the device.
+ */
+ private boolean writeEntry(PiTableEntry entry,
+ WriteOperationType p4Operation) {
+ try {
+ if (client.writeTableEntries(
+ newArrayList(entry), p4Operation, pipeconf).get()) {
+ return true;
+ } else {
+ log.warn("Unable to {} table entry in {}: {}",
+ p4Operation.name(), deviceId, entry);
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ log.warn("Exception while performing {} table entry operation:",
+ p4Operation, e);
+ }
+ return false;
+ }
+
+ private void updateStores(PiTableEntryHandle handle,
+ PiTableEntry entry,
+ FlowRule rule,
+ WriteOperationType p4Operation) {
+ switch (p4Operation) {
+ case INSERT:
+ case MODIFY:
+ tableMirror.put(handle, entry);
+ translator.learn(handle, new PiTranslatedEntity<>(rule, entry, handle));
+ break;
+ case DELETE:
+ tableMirror.remove(handle);
+ translator.forget(handle);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unknown operation " + p4Operation.name());
+ }
+ }
+
+ private Map<PiTableEntry, PiCounterCellData> readEntryCounters(
+ PiCounterId counterId, Collection<PiTableEntry> tableEntries) {
+ Collection<PiCounterCellData> cellDatas;
+ try {
+ if (readAllDirectCounters) {
+ cellDatas = client.readAllCounterCells(
+ singleton(counterId), pipeconf).get();
+ } else {
+ Set<PiCounterCellId> cellIds = tableEntries.stream()
+ .map(entry -> PiCounterCellId.ofDirect(counterId, entry))
+ .collect(Collectors.toSet());
+ cellDatas = client.readCounterCells(cellIds, pipeconf).get();
+ }
+ return cellDatas.stream()
+ .collect(Collectors.toMap(c -> c.cellId().tableEntry(), c -> c));
+ } catch (InterruptedException | ExecutionException e) {
+ if (!(e.getCause() instanceof StatusRuntimeException)) {
+ // gRPC errors are logged in the client.
+ log.error("Exception while reading counter '{}' from {}: {}",
+ counterId, deviceId, e);
+ }
+ return Collections.emptyMap();
+ }
}
enum Operation {