More robust P4Runtime group handling
This patch solves the PENDING_UPDATE and PENDING_ADD_RETRY issue
observed on the ONS EU topology.
The P4Runtime action profile group handling has been re-implemented to
be robust against inconsistencies of the device mirror, which is now
periodically synchronized with the device state. Similarly, we implement
a routine in the P4RuntimeClient to cleanup unused action profile
members.
This patch includes also:
- Refactor PI handle classes to allow creating handles without the
entity instance
- Use list instead of collections in P4RuntimeClient methods, as order
of updates sent and/or entities received from the device is important
Change-Id: I2e7964ce90f43d66680131b47ab52aca32ab55d2
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeActionGroupProgrammable.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeActionGroupProgrammable.java
new file mode 100644
index 0000000..5ee965d
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeActionGroupProgrammable.java
@@ -0,0 +1,436 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Striped;
+import org.onlab.util.SharedExecutors;
+import org.onosproject.drivers.p4runtime.mirror.P4RuntimeActionProfileMemberMirror;
+import org.onosproject.drivers.p4runtime.mirror.P4RuntimeGroupMirror;
+import org.onosproject.drivers.p4runtime.mirror.TimedEntry;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.group.DefaultGroup;
+import org.onosproject.net.group.DefaultGroupDescription;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupDescription;
+import org.onosproject.net.group.GroupOperation;
+import org.onosproject.net.group.GroupOperations;
+import org.onosproject.net.group.GroupProgrammable;
+import org.onosproject.net.group.GroupStore;
+import org.onosproject.net.pi.model.PiActionId;
+import org.onosproject.net.pi.model.PiActionProfileId;
+import org.onosproject.net.pi.model.PiActionProfileModel;
+import org.onosproject.net.pi.runtime.PiAction;
+import org.onosproject.net.pi.runtime.PiActionGroup;
+import org.onosproject.net.pi.runtime.PiActionGroupHandle;
+import org.onosproject.net.pi.runtime.PiActionGroupMember;
+import org.onosproject.net.pi.runtime.PiActionGroupMemberHandle;
+import org.onosproject.net.pi.runtime.PiActionGroupMemberId;
+import org.onosproject.net.pi.service.PiGroupTranslator;
+import org.onosproject.net.pi.service.PiTranslatedEntity;
+import org.onosproject.net.pi.service.PiTranslationException;
+import org.onosproject.p4runtime.api.P4RuntimeClient;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.DELETE;
+import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.INSERT;
+import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.MODIFY;
+
+/**
+ * Implementation of GroupProgrammable to handle action profile groups in
+ * P4Runtime.
+ */
+public class P4RuntimeActionGroupProgrammable
+ extends AbstractP4RuntimeHandlerBehaviour
+ implements GroupProgrammable {
+
+ // If true, we avoid querying the device and return what's already known by
+ // the ONOS store.
+ private static final String READ_ACTION_GROUPS_FROM_MIRROR = "actionGroupReadFromMirror";
+ private static final boolean DEFAULT_READ_ACTION_GROUPS_FROM_MIRROR = false;
+
+ protected GroupStore groupStore;
+ private P4RuntimeGroupMirror groupMirror;
+ private P4RuntimeActionProfileMemberMirror memberMirror;
+ private PiGroupTranslator groupTranslator;
+
+ // Needed to synchronize operations over the same group.
+ private static final Striped<Lock> STRIPED_LOCKS = Striped.lock(30);
+
+ @Override
+ protected boolean setupBehaviour() {
+ if (!super.setupBehaviour()) {
+ return false;
+ }
+ groupMirror = this.handler().get(P4RuntimeGroupMirror.class);
+ memberMirror = this.handler().get(P4RuntimeActionProfileMemberMirror.class);
+ groupStore = handler().get(GroupStore.class);
+ groupTranslator = piTranslationService.groupTranslator();
+ return true;
+ }
+
+ @Override
+ public void performGroupOperation(DeviceId deviceId,
+ GroupOperations groupOps) {
+ if (!setupBehaviour()) {
+ return;
+ }
+
+ groupOps.operations().stream()
+ .filter(op -> !op.groupType().equals(GroupDescription.Type.ALL))
+ .forEach(op -> {
+ // ONOS-7785 We need app cookie (action profile id) from the group
+ Group groupOnStore = groupStore.getGroup(deviceId, op.groupId());
+ GroupDescription groupDesc = new DefaultGroupDescription(
+ deviceId, op.groupType(), op.buckets(), groupOnStore.appCookie(),
+ op.groupId().id(), groupOnStore.appId());
+ DefaultGroup groupToApply = new DefaultGroup(op.groupId(), groupDesc);
+ processGroupOperation(groupToApply, op.opType());
+ });
+ }
+
+ @Override
+ public Collection<Group> getGroups() {
+ if (!setupBehaviour()) {
+ return Collections.emptyList();
+ }
+ return getActionGroups();
+ }
+
+ private Collection<Group> getActionGroups() {
+
+ if (driverBoolProperty(READ_ACTION_GROUPS_FROM_MIRROR,
+ DEFAULT_READ_ACTION_GROUPS_FROM_MIRROR)) {
+ return getActionGroupsFromMirror();
+ }
+
+ final Collection<PiActionProfileId> actionProfileIds = pipeconf.pipelineModel()
+ .actionProfiles()
+ .stream()
+ .map(PiActionProfileModel::id)
+ .collect(Collectors.toList());
+ final List<PiActionGroup> groupsOnDevice = actionProfileIds.stream()
+ .flatMap(this::streamGroupsFromDevice)
+ .collect(Collectors.toList());
+ final Set<PiActionGroupMemberHandle> membersOnDevice = actionProfileIds
+ .stream()
+ .flatMap(actProfId -> getMembersFromDevice(actProfId)
+ .stream()
+ .map(memberId -> PiActionGroupMemberHandle.of(
+ deviceId, actProfId, memberId)))
+ .collect(Collectors.toSet());
+
+ if (groupsOnDevice.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ // Sync mirrors.
+ syncGroupMirror(groupsOnDevice);
+ syncMemberMirror(membersOnDevice);
+
+ final List<Group> result = Lists.newArrayList();
+ final List<PiActionGroup> inconsistentGroups = Lists.newArrayList();
+ final List<PiActionGroup> validGroups = Lists.newArrayList();
+
+ for (PiActionGroup piGroup : groupsOnDevice) {
+ final Group pdGroup = forgeGroupEntry(piGroup);
+ if (pdGroup == null) {
+ // Entry is on device but unknown to translation service or
+ // device mirror. Inconsistent. Mark for removal.
+ inconsistentGroups.add(piGroup);
+ } else {
+ validGroups.add(piGroup);
+ result.add(pdGroup);
+ }
+ }
+
+ // Trigger clean up of inconsistent groups and members. This will also
+ // remove all members that are not used by any group, and update the
+ // mirror accordingly.
+ final Set<PiActionGroupMemberHandle> membersToKeep = validGroups.stream()
+ .flatMap(g -> g.members().stream())
+ .map(m -> PiActionGroupMemberHandle.of(deviceId, m))
+ .collect(Collectors.toSet());
+ final Set<PiActionGroupMemberHandle> inconsistentMembers = Sets.difference(
+ membersOnDevice, membersToKeep);
+ SharedExecutors.getSingleThreadExecutor().execute(
+ () -> cleanUpInconsistentGroupsAndMembers(
+ inconsistentGroups, inconsistentMembers));
+
+ return result;
+ }
+
+ private void syncGroupMirror(Collection<PiActionGroup> groups) {
+ Map<PiActionGroupHandle, PiActionGroup> handleMap = Maps.newHashMap();
+ groups.forEach(g -> handleMap.put(PiActionGroupHandle.of(deviceId, g), g));
+ groupMirror.sync(deviceId, handleMap);
+ }
+
+ private void syncMemberMirror(Collection<PiActionGroupMemberHandle> memberHandles) {
+ Map<PiActionGroupMemberHandle, PiActionGroupMember> handleMap = Maps.newHashMap();
+ memberHandles.forEach(handle -> handleMap.put(
+ handle, dummyMember(handle.actionProfileId(), handle.memberId())));
+ memberMirror.sync(deviceId, handleMap);
+ }
+
+ private Collection<Group> getActionGroupsFromMirror() {
+ return groupMirror.getAll(deviceId).stream()
+ .map(TimedEntry::entry)
+ .map(this::forgeGroupEntry)
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+ }
+
+ private void cleanUpInconsistentGroupsAndMembers(Collection<PiActionGroup> groupsToRemove,
+ Collection<PiActionGroupMemberHandle> membersToRemove) {
+ if (!groupsToRemove.isEmpty()) {
+ log.warn("Found {} inconsistent action profile groups on {}, removing them...",
+ groupsToRemove.size(), deviceId);
+ groupsToRemove.forEach(piGroup -> {
+ log.debug(piGroup.toString());
+ processGroup(piGroup, null, Operation.REMOVE);
+ });
+ }
+ if (!membersToRemove.isEmpty()) {
+ log.warn("Found {} inconsistent action profile members on {}, removing them...",
+ membersToRemove.size(), deviceId);
+ // FIXME: implement client call to remove members from multiple
+ // action profiles in one shot.
+ final ListMultimap<PiActionProfileId, PiActionGroupMemberId>
+ membersByActProfId = ArrayListMultimap.create();
+ membersToRemove.forEach(m -> membersByActProfId.put(
+ m.actionProfileId(), m.memberId()));
+ membersByActProfId.keySet().forEach(actProfId -> {
+ List<PiActionGroupMemberId> removedMembers = getFutureWithDeadline(
+ client.removeActionProfileMembers(
+ actProfId, membersByActProfId.get(actProfId), pipeconf),
+ "cleaning up action profile members", Collections.emptyList());
+ // Update member mirror.
+ removedMembers.stream()
+ .map(id -> PiActionGroupMemberHandle.of(deviceId, actProfId, id))
+ .forEach(memberMirror::remove);
+ });
+ }
+ }
+
+ private Stream<PiActionGroup> streamGroupsFromDevice(PiActionProfileId actProfId) {
+ // TODO: implement P4Runtime client call to read all groups with one call
+ // Good if pipeline has multiple action profiles.
+ final Collection<PiActionGroup> groups = getFutureWithDeadline(
+ client.dumpGroups(actProfId, pipeconf),
+ "dumping groups", Collections.emptyList());
+ return groups.stream();
+ }
+
+ private List<PiActionGroupMemberId> getMembersFromDevice(PiActionProfileId actProfId) {
+ // TODO: implement P4Runtime client call to read all members with one call
+ // Good if pipeline has multiple action profiles.
+ return getFutureWithDeadline(
+ client.dumpActionProfileMemberIds(actProfId, pipeconf),
+ "dumping action profile ids", Collections.emptyList());
+ }
+
+ private Group forgeGroupEntry(PiActionGroup piGroup) {
+ final PiActionGroupHandle handle = PiActionGroupHandle.of(deviceId, piGroup);
+ final Optional<PiTranslatedEntity<Group, PiActionGroup>>
+ translatedEntity = groupTranslator.lookup(handle);
+ final TimedEntry<PiActionGroup> timedEntry = groupMirror.get(handle);
+ // Is entry consistent with our state?
+ if (!translatedEntity.isPresent()) {
+ log.warn("Group handle not found in translation store: {}", handle);
+ return null;
+ }
+ if (!translatedEntity.get().translated().equals(piGroup)) {
+ log.warn("Group obtained from device {} is different from the one in" +
+ "translation store: device={}, store={}",
+ deviceId, piGroup, translatedEntity.get().translated());
+ return null;
+ }
+ if (timedEntry == null) {
+ log.warn("Group handle not found in device mirror: {}", handle);
+ return null;
+ }
+ return addedGroup(translatedEntity.get().original(), timedEntry.lifeSec());
+ }
+
+ private Group addedGroup(Group original, long life) {
+ final DefaultGroup forgedGroup = new DefaultGroup(original.id(), original);
+ forgedGroup.setState(Group.GroupState.ADDED);
+ forgedGroup.setLife(life);
+ return forgedGroup;
+ }
+
+ private void processGroupOperation(Group pdGroup, GroupOperation.Type opType) {
+ final PiActionGroup piGroup;
+ try {
+ piGroup = groupTranslator.translate(pdGroup, pipeconf);
+ } catch (PiTranslationException e) {
+ log.warn("Unable to translate group, aborting {} operation: {} [{}]",
+ opType, e.getMessage(), pdGroup);
+ return;
+ }
+ final Operation operation = opType.equals(GroupOperation.Type.DELETE)
+ ? Operation.REMOVE : Operation.APPLY;
+ processGroup(piGroup, pdGroup, operation);
+ }
+
+ private void processGroup(PiActionGroup groupToApply,
+ Group pdGroup,
+ Operation operation) {
+ final PiActionGroupHandle handle = PiActionGroupHandle.of(deviceId, groupToApply);
+ STRIPED_LOCKS.get(handle).lock();
+ try {
+ switch (operation) {
+ case APPLY:
+ if (applyGroupWithMembersOrNothing(groupToApply, handle)) {
+ groupTranslator.learn(handle, new PiTranslatedEntity<>(
+ pdGroup, groupToApply, handle));
+ }
+ return;
+ case REMOVE:
+ if (deleteGroup(groupToApply, handle)) {
+ groupTranslator.forget(handle);
+ }
+ return;
+ default:
+ log.error("Unknwon group operation type {}, cannot process group", operation);
+ break;
+ }
+ } finally {
+ STRIPED_LOCKS.get(handle).unlock();
+ }
+ }
+
+ private boolean applyGroupWithMembersOrNothing(PiActionGroup group, PiActionGroupHandle handle) {
+ // First apply members, then group, if fails, delete members.
+ if (!applyAllMembersOrNothing(group.members())) {
+ return false;
+ }
+ if (!applyGroup(group, handle)) {
+ deleteMembers(group.members());
+ return false;
+ }
+ return true;
+ }
+
+ private boolean applyGroup(PiActionGroup group, PiActionGroupHandle handle) {
+ final P4RuntimeClient.WriteOperationType opType =
+ groupMirror.get(handle) == null ? INSERT : MODIFY;
+ final boolean success = getFutureWithDeadline(
+ client.writeActionGroup(group, opType, pipeconf),
+ "performing action profile group " + opType, false);
+ if (success) {
+ groupMirror.put(handle, group);
+ }
+ return success;
+ }
+
+ private boolean deleteGroup(PiActionGroup group, PiActionGroupHandle handle) {
+ final boolean success = getFutureWithDeadline(
+ client.writeActionGroup(group, DELETE, pipeconf),
+ "performing action profile group " + DELETE, false);
+ if (success) {
+ groupMirror.remove(handle);
+ }
+ return success;
+ }
+
+ private boolean applyAllMembersOrNothing(Collection<PiActionGroupMember> members) {
+ Collection<PiActionGroupMember> appliedMembers = applyMembers(members);
+ if (appliedMembers.size() == members.size()) {
+ return true;
+ } else {
+ deleteMembers(appliedMembers);
+ return false;
+ }
+ }
+
+ private Collection<PiActionGroupMember> applyMembers(
+ Collection<PiActionGroupMember> members) {
+ return members.stream()
+ .filter(this::applyMember)
+ .collect(Collectors.toList());
+ }
+
+ private boolean applyMember(PiActionGroupMember member) {
+ // If exists, modify, otherwise insert
+ final PiActionGroupMemberHandle handle = PiActionGroupMemberHandle.of(
+ deviceId, member);
+ final P4RuntimeClient.WriteOperationType opType =
+ memberMirror.get(handle) == null ? INSERT : MODIFY;
+ final boolean success = getFutureWithDeadline(
+ client.writeActionGroupMembers(Collections.singletonList(member),
+ opType, pipeconf),
+ "performing action profile member " + opType, false);
+ if (success) {
+ memberMirror.put(handle, dummyMember(member.actionProfile(), member.id()));
+ }
+ return success;
+ }
+
+ private void deleteMembers(Collection<PiActionGroupMember> members) {
+ members.forEach(this::deleteMember);
+ }
+
+ private void deleteMember(PiActionGroupMember member) {
+ final PiActionGroupMemberHandle handle = PiActionGroupMemberHandle.of(
+ deviceId, member);
+ final boolean success = getFutureWithDeadline(
+ client.writeActionGroupMembers(Collections.singletonList(member),
+ DELETE, pipeconf),
+ "performing action profile member " + DELETE, false);
+ if (success) {
+ memberMirror.remove(handle);
+ }
+ }
+
+ // FIXME: this is nasty, we have to rely on a dummy member of the mirror
+ // because the PiActionGroupMember abstraction is broken, since it includes
+ // attributes that are not part of a P4Runtime member, e.g. weight.
+ // We should remove weight from the class, and have client methods that
+ // return the full PiActionGroupMember, not just the IDs. Also the naming
+ // "ActionGroupMember" is wrong since it makes believe that members can
+ // exists only inside a group, which is not true.
+ private PiActionGroupMember dummyMember(
+ PiActionProfileId actionProfileId, PiActionGroupMemberId memberId) {
+ return PiActionGroupMember.builder()
+ .forActionProfile(actionProfileId)
+ .withId(memberId)
+ .withAction(PiAction.builder()
+ .withId(PiActionId.of("dummy"))
+ .build())
+ .build();
+ }
+
+ enum Operation {
+ APPLY, REMOVE
+ }
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
index 85a87fe..d0acdee 100644
--- a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java
@@ -140,6 +140,11 @@
streamEntries(), streamDefaultEntries())
// Ignore entries from constant tables.
.filter(e -> !tableIsConstant(e.table()))
+ // Device implementation might return duplicate entries. For
+ // example if reading only default ones is not supported and
+ // non-default entries are returned, by using distinct() we are
+ // robust against that possibility.
+ .distinct()
.collect(Collectors.toList());
if (deviceEntries.isEmpty()) {
@@ -148,7 +153,6 @@
// Synchronize mirror with the device state.
syncMirror(deviceEntries);
- // Read table direct counters for non default-entries (if any).
// TODO: ONOS-7596 read counters with table entries
final Map<PiTableEntry, PiCounterCellData> counterCellMap =
readEntryCounters(deviceEntries);
@@ -229,7 +233,12 @@
log.warn("Table entry handle not found in translation store: {}", handle);
return null;
}
-
+ if (!translatedEntity.get().translated().equals(entry)) {
+ log.warn("Table entry obtained from device {} is different from " +
+ "one in in translation store: device={}, store={}",
+ deviceId, entry, translatedEntity.get().translated());
+ return null;
+ }
if (timedEntry == null) {
log.warn("Table entry handle not found in device mirror: {}", handle);
return null;
@@ -460,6 +469,7 @@
cellDatas = Collections.emptyList();
} else {
Set<PiCounterCellId> cellIds = tableEntries.stream()
+ // Ignore counter for default entry.
.filter(e -> !e.isDefaultAction())
.filter(e -> tableHasCounter(e.table()))
.map(PiCounterCellId::ofDirect)
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java
index f53dca8..84678dc 100644
--- a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2017-present Open Networking Foundation
+ * Copyright 2018-present Open Networking Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,545 +18,92 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Striped;
-import org.onlab.util.SharedExecutors;
-import org.onosproject.drivers.p4runtime.mirror.P4RuntimeGroupMirror;
-import org.onosproject.drivers.p4runtime.mirror.P4RuntimeMulticastGroupMirror;
-import org.onosproject.drivers.p4runtime.mirror.TimedEntry;
import org.onosproject.net.DeviceId;
-import org.onosproject.net.group.DefaultGroup;
-import org.onosproject.net.group.DefaultGroupDescription;
+import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.group.Group;
import org.onosproject.net.group.GroupDescription;
import org.onosproject.net.group.GroupOperation;
import org.onosproject.net.group.GroupOperations;
import org.onosproject.net.group.GroupProgrammable;
-import org.onosproject.net.group.GroupStore;
-import org.onosproject.net.pi.model.PiActionProfileId;
-import org.onosproject.net.pi.model.PiActionProfileModel;
-import org.onosproject.net.pi.runtime.PiActionGroup;
-import org.onosproject.net.pi.runtime.PiActionGroupHandle;
-import org.onosproject.net.pi.runtime.PiActionGroupMember;
-import org.onosproject.net.pi.runtime.PiMulticastGroupEntry;
-import org.onosproject.net.pi.runtime.PiMulticastGroupEntryHandle;
-import org.onosproject.net.pi.service.PiGroupTranslator;
-import org.onosproject.net.pi.service.PiMulticastGroupTranslator;
-import org.onosproject.net.pi.service.PiTranslatedEntity;
-import org.onosproject.net.pi.service.PiTranslationException;
-import org.onosproject.p4runtime.api.P4RuntimeClient;
import org.slf4j.Logger;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.locks.Lock;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
import static com.google.common.base.Preconditions.checkArgument;
-import static java.lang.String.format;
-import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.DELETE;
-import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.INSERT;
-import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.MODIFY;
import static org.slf4j.LoggerFactory.getLogger;
/**
- * Implementation of the group programmable behaviour for P4Runtime.
- * <p>
- * This implementation distinguishes between ALL groups, and other types. ALL
- * groups are handled via PRE multicast group programming, while other types are
- * handled via action profile group programming.
+ * Implementation of GroupProgrammable for P4Runtime devices that uses two
+ * different implementation of the same behavior to handle both action profile
+ * groups and multicast groups.
*/
public class P4RuntimeGroupProgrammable
- extends AbstractP4RuntimeHandlerBehaviour
- implements GroupProgrammable {
+ extends AbstractHandlerBehaviour implements GroupProgrammable {
- private static final String ACT_GRP_MEMS_STR = "action group members";
- private static final String DELETE_STR = "delete";
- private static final String ACT_GRP_STR = "action group";
- private static final String INSERT_STR = "insert";
- private static final String MODIFY_STR = "modify";
+ private final Logger log = getLogger(this.getClass());
- private static final Logger log = getLogger(P4RuntimeGroupProgrammable.class);
-
- // If true, we ignore re-installing groups that are already known in the
- // device mirror.
- private static final String CHECK_MIRROR_BEFORE_UPDATE = "checkMirrorBeforeUpdate";
- private static final boolean DEFAULT_CHECK_MIRROR_BEFORE_UPDATE = true;
-
- // If true, we avoid querying the device and return what's already known by
- // the ONOS store.
- private static final String READ_ACTION_GROUPS_FROM_MIRROR = "actionGroupReadFromMirror";
- private static final boolean DEFAULT_READ_ACTION_GROUPS_FROM_MIRROR = false;
-
- protected GroupStore groupStore;
- private P4RuntimeGroupMirror groupMirror;
- private PiGroupTranslator groupTranslator;
- private P4RuntimeMulticastGroupMirror mcGroupMirror;
- private PiMulticastGroupTranslator mcGroupTranslator;
-
- // Needed to synchronize operations over the same group.
- private static final Striped<Lock> STRIPED_LOCKS = Striped.lock(30);
-
- @Override
- protected boolean setupBehaviour() {
- if (!super.setupBehaviour()) {
- return false;
+ private void doPerformGroupOperation(DeviceId deviceId, GroupOperations groupOps) {
+ // TODO: fix GroupProgrammable API, passing the device ID is ambiguous
+ checkArgument(deviceId.equals(data().deviceId()),
+ "passed deviceId must be the same assigned to this behavior");
+ final List<GroupOperation> actionGroups = Lists.newArrayList();
+ final List<GroupOperation> multicastGroups = Lists.newArrayList();
+ groupOps.operations().forEach(op -> {
+ if (op.groupType().equals(GroupDescription.Type.ALL)) {
+ multicastGroups.add(op);
+ } else {
+ actionGroups.add(op);
+ }
+ });
+ if (!actionGroups.isEmpty()) {
+ actionProgrammable().performGroupOperation(
+ deviceId, new GroupOperations(actionGroups));
}
- groupMirror = this.handler().get(P4RuntimeGroupMirror.class);
- mcGroupMirror = this.handler().get(P4RuntimeMulticastGroupMirror.class);
- groupStore = handler().get(GroupStore.class);
- groupTranslator = piTranslationService.groupTranslator();
- mcGroupTranslator = piTranslationService.multicastGroupTranslator();
- return true;
+ if (!multicastGroups.isEmpty()) {
+ multicastProgrammable().performGroupOperation(
+ deviceId, new GroupOperations(multicastGroups));
+ }
+ }
+
+ private Collection<Group> doGetGroups() {
+ return new ImmutableList.Builder<Group>()
+ .addAll(actionProgrammable().getGroups())
+ .addAll(multicastProgrammable().getGroups())
+ .build();
+ }
+
+ private P4RuntimeActionGroupProgrammable actionProgrammable() {
+ P4RuntimeActionGroupProgrammable prog = new P4RuntimeActionGroupProgrammable();
+ prog.setData(data());
+ prog.setHandler(handler());
+ return prog;
+ }
+
+ private P4RuntimeMulticastGroupProgrammable multicastProgrammable() {
+ P4RuntimeMulticastGroupProgrammable prog = new P4RuntimeMulticastGroupProgrammable();
+ prog.setData(data());
+ prog.setHandler(handler());
+ return prog;
}
@Override
- public void performGroupOperation(DeviceId deviceId,
- GroupOperations groupOps) {
- if (!setupBehaviour()) {
- return;
+ public void performGroupOperation(DeviceId deviceId, GroupOperations groupOps) {
+ try {
+ doPerformGroupOperation(deviceId, groupOps);
+ } catch (Throwable ex) {
+ log.error("Unhandled exception on performGroupOperation", ex);
}
-
- // TODO: fix GroupProgrammable API, passing the device ID is ambiguous
- checkArgument(deviceId.equals(this.deviceId),
- "passed deviceId must be the same assigned to this behavior");
-
- groupOps.operations().forEach(op -> {
- // ONOS-7785 We need app cookie (action profile id) from the group
- Group groupOnStore = groupStore.getGroup(deviceId, op.groupId());
- GroupDescription groupDesc = new DefaultGroupDescription(deviceId,
- op.groupType(),
- op.buckets(),
- groupOnStore.appCookie(),
- op.groupId().id(),
- groupOnStore.appId());
- DefaultGroup groupToApply = new DefaultGroup(op.groupId(), groupDesc);
- if (op.groupType().equals(GroupDescription.Type.ALL)) {
- processMcGroupOp(groupToApply, op.opType());
- } else {
-
- processActionGroupOp(groupToApply, op.opType());
- }
- });
}
@Override
public Collection<Group> getGroups() {
- if (!setupBehaviour()) {
+ try {
+ return doGetGroups();
+ } catch (Throwable ex) {
+ log.error("Unhandled exception on getGroups", ex);
return Collections.emptyList();
}
- return new ImmutableList.Builder<Group>()
- .addAll(getActionGroups())
- .addAll(getMcGroups()).build();
- }
-
- private Collection<Group> getActionGroups() {
-
- if (driverBoolProperty(READ_ACTION_GROUPS_FROM_MIRROR,
- DEFAULT_READ_ACTION_GROUPS_FROM_MIRROR)) {
- return getActionGroupsFromMirror();
- }
-
- final Collection<PiActionGroup> piGroups = pipeconf.pipelineModel()
- .actionProfiles()
- .stream()
- .map(PiActionProfileModel::id)
- .flatMap(this::streamPiGroupsFromDevice)
- .collect(Collectors.toList());
-
- if (piGroups.isEmpty()) {
- return Collections.emptyList();
- }
-
- final List<Group> result = Lists.newArrayList();
- final List<PiActionGroup> inconsistentGroups = Lists.newArrayList();
-
- for (PiActionGroup piGroupOnDevice : piGroups) {
- final Group group = forgeGroupEntry(piGroupOnDevice);
- if (group == null) {
- // Entry is on device but unknown to translation service or
- // device mirror. Inconsistent. Mark for removal.
- inconsistentGroups.add(piGroupOnDevice);
- } else {
- result.add(group);
- }
- }
- // Trigger clean up of inconsistent entries (is any).
- // TODO: make this behaviour configurable, in some cases it's fine for
- // the device to have groups that were not installed by us.
- if (!inconsistentGroups.isEmpty()) {
- SharedExecutors.getSingleThreadExecutor().execute(
- () -> cleanUpInconsistentGroups(inconsistentGroups));
- }
- return result;
- }
-
- private Collection<Group> getActionGroupsFromMirror() {
- return groupMirror.getAll(deviceId).stream()
- .map(TimedEntry::entry)
- .map(this::forgeGroupEntry)
- .filter(Objects::nonNull)
- .collect(Collectors.toList());
- }
-
- private void cleanUpInconsistentGroups(Collection<PiActionGroup> piGroups) {
- log.warn("Found {} inconsistent groups on {}, removing them...",
- piGroups.size(), deviceId);
- piGroups.forEach(piGroup -> {
- log.debug(piGroup.toString());
- // Per-piGroup lock.
- final PiActionGroupHandle handle = PiActionGroupHandle.of(deviceId, piGroup);
- STRIPED_LOCKS.get(handle).lock();
- try {
- processActionGroup(handle, piGroup, null, null,
- GroupOperation.Type.DELETE);
- } finally {
- STRIPED_LOCKS.get(handle).unlock();
- }
- });
- }
-
- private Collection<Group> getMcGroups() {
- // TODO: missing support for reading multicast groups is ready in PI/Stratum.
- return getMcGroupsFromMirror();
- }
-
- private Collection<Group> getMcGroupsFromMirror() {
- return mcGroupMirror.getAll(deviceId).stream()
- .map(TimedEntry::entry)
- .map(this::forgeMcGroupEntry)
- .filter(Objects::nonNull)
- .collect(Collectors.toList());
- }
-
- private void processActionGroupOp(Group pdGroup, GroupOperation.Type opType) {
- final PiActionGroup piGroup;
- try {
- piGroup = groupTranslator.translate(pdGroup, pipeconf);
- } catch (PiTranslationException e) {
- log.warn("Unable to translate group, aborting {} operation: {} [{}]",
- opType, e.getMessage(), pdGroup);
- return;
- }
- final PiActionGroupHandle handle = PiActionGroupHandle.of(deviceId, piGroup);
- final PiActionGroup groupOnDevice = groupMirror.get(handle) == null
- ? null : groupMirror.get(handle).entry();
- // Per-piGroup lock.
- final Lock lock = STRIPED_LOCKS.get(handle);
- lock.lock();
- try {
- processActionGroup(handle, piGroup,
- groupOnDevice, pdGroup, opType);
- } finally {
- lock.unlock();
- }
- }
-
- private void processMcGroupOp(Group pdGroup, GroupOperation.Type opType) {
- final PiMulticastGroupEntry mcGroup;
- try {
- mcGroup = mcGroupTranslator.translate(pdGroup, pipeconf);
- } catch (PiTranslationException e) {
- log.warn("Unable to translate multicast group, aborting {} operation: {} [{}]",
- opType, e.getMessage(), pdGroup);
- return;
- }
- final PiMulticastGroupEntryHandle handle = PiMulticastGroupEntryHandle.of(
- deviceId, mcGroup);
- final PiMulticastGroupEntry groupOnDevice = mcGroupMirror.get(handle) == null
- ? null
- : mcGroupMirror.get(handle).entry();
- final Lock lock = STRIPED_LOCKS.get(handle);
- lock.lock();
- try {
- processMcGroup(handle, mcGroup,
- groupOnDevice, pdGroup, opType);
- } finally {
- lock.unlock();
- }
- }
-
- private void processActionGroup(PiActionGroupHandle handle,
- PiActionGroup groupToApply,
- PiActionGroup groupOnDevice,
- Group pdGroup, GroupOperation.Type operationType) {
- switch (operationType) {
- case ADD:
- if (groupOnDevice != null) {
- log.warn("Requested to ADD group {} on {}, but a group " +
- "with the same ID already exists, will " +
- "MODIFY instead",
- groupToApply.id(), deviceId);
- log.debug("To apply: {}", groupToApply);
- log.debug("On device: {}", groupOnDevice);
- processActionGroup(handle, groupToApply, groupOnDevice,
- pdGroup, GroupOperation.Type.MODIFY);
- return;
- }
- if (writeGroupToDevice(groupToApply)) {
- groupMirror.put(handle, groupToApply);
- groupTranslator.learn(handle, new PiTranslatedEntity<>(
- pdGroup, groupToApply, handle));
- }
- return;
- case MODIFY:
- if (groupOnDevice == null) {
- log.warn("Requested to MODIFY group {} on {}, but no " +
- "such group exists on the device, " +
- "will ADD instead",
- groupToApply.id(), deviceId);
- processActionGroup(handle, groupToApply, null,
- pdGroup, GroupOperation.Type.ADD);
- return;
- }
- if (driverBoolProperty(CHECK_MIRROR_BEFORE_UPDATE,
- DEFAULT_CHECK_MIRROR_BEFORE_UPDATE)
- && groupOnDevice.equals(groupToApply)) {
- // Group on device has the same members, ignore operation.
- return;
- }
- if (modifyGroupFromDevice(groupToApply, groupOnDevice)) {
- groupMirror.put(handle, groupToApply);
- groupTranslator.learn(handle, new PiTranslatedEntity<>(
- pdGroup, groupToApply, handle));
- }
- return;
- case DELETE:
- if (deleteGroupFromDevice(groupToApply)) {
- groupMirror.remove(handle);
- groupTranslator.forget(handle);
- }
- break;
- default:
- log.error("Unknwon group operation type {}, cannot process group", operationType);
- break;
- }
- }
-
- private void processMcGroup(PiMulticastGroupEntryHandle handle,
- PiMulticastGroupEntry groupToApply,
- PiMulticastGroupEntry groupOnDevice,
- Group pdGroup, GroupOperation.Type opType) {
- switch (opType) {
- case ADD:
- robustMcGroupAdd(handle, groupToApply, pdGroup);
- return;
- case MODIFY:
- // Since reading multicast groups is not supported yet on
- // PI/Stratum, we cannot trust groupOnDevic) as we don't have a
- // mechanism to enforce consistency of the mirror with the
- // device state.
- // if (driverBoolProperty(CHECK_MIRROR_BEFORE_UPDATE,
- // DEFAULT_CHECK_MIRROR_BEFORE_UPDATE)
- // && p4OpType == MODIFY
- // && groupOnDevice != null
- // && groupOnDevice.equals(groupToApply)) {
- // // Ignore.
- // return;
- // }
- robustMcGroupModify(handle, groupToApply, pdGroup);
- return;
- case DELETE:
- mcGroupApply(handle, groupToApply, pdGroup, DELETE);
- return;
- default:
- log.error("Unknown group operation type {}, " +
- "cannot process multicast group", opType);
- }
- }
-
- private boolean writeMcGroupOnDevice(PiMulticastGroupEntry group, P4RuntimeClient.WriteOperationType opType) {
- return getFutureWithDeadline(
- client.writePreMulticastGroupEntries(
- Collections.singleton(group), opType),
- "performing multicast group " + opType, false);
- }
-
- private boolean mcGroupApply(PiMulticastGroupEntryHandle handle,
- PiMulticastGroupEntry piGroup,
- Group pdGroup,
- P4RuntimeClient.WriteOperationType opType) {
- switch (opType) {
- case DELETE:
- if (writeMcGroupOnDevice(piGroup, DELETE)) {
- mcGroupMirror.remove(handle);
- mcGroupTranslator.forget(handle);
- return true;
- } else {
- return false;
- }
- case INSERT:
- case MODIFY:
- if (writeMcGroupOnDevice(piGroup, opType)) {
- mcGroupMirror.put(handle, piGroup);
- mcGroupTranslator.learn(handle, new PiTranslatedEntity<>(
- pdGroup, piGroup, handle));
- return true;
- } else {
- return false;
- }
- default:
- log.warn("Unknown operation type {}, cannot apply group", opType);
- return false;
- }
- }
-
- private void robustMcGroupAdd(PiMulticastGroupEntryHandle handle,
- PiMulticastGroupEntry piGroup,
- Group pdGroup) {
- if (mcGroupApply(handle, piGroup, pdGroup, INSERT)) {
- return;
- }
- // Try to delete (perhaps it already exists) and re-add...
- mcGroupApply(handle, piGroup, pdGroup, DELETE);
- mcGroupApply(handle, piGroup, pdGroup, INSERT);
- }
-
- private void robustMcGroupModify(PiMulticastGroupEntryHandle handle,
- PiMulticastGroupEntry piGroup,
- Group pdGroup) {
- if (mcGroupApply(handle, piGroup, pdGroup, MODIFY)) {
- return;
- }
- // Not sure for which reason it cannot be modified, so try to delete and insert instead...
- mcGroupApply(handle, piGroup, pdGroup, DELETE);
- mcGroupApply(handle, piGroup, pdGroup, INSERT);
- }
-
- private boolean modifyGroupFromDevice(PiActionGroup groupToApply, PiActionGroup groupOnDevice) {
- PiActionProfileId groupProfileId = groupToApply.actionProfileId();
- Collection<PiActionGroupMember> membersToRemove = Sets.newHashSet(groupOnDevice.members());
- membersToRemove.removeAll(groupToApply.members());
- Collection<PiActionGroupMember> membersToAdd = Sets.newHashSet(groupToApply.members());
- membersToAdd.removeAll(groupOnDevice.members());
-
- if (!membersToAdd.isEmpty() &&
- !completeFuture(client.writeActionGroupMembers(groupProfileId, membersToAdd, INSERT, pipeconf),
- ACT_GRP_MEMS_STR, INSERT_STR)) {
- // remove what we added
- completeFuture(client.writeActionGroupMembers(groupProfileId, membersToAdd, DELETE, pipeconf),
- ACT_GRP_MEMS_STR, INSERT_STR);
- return false;
- }
-
- if (!completeFuture(client.writeActionGroup(groupToApply, MODIFY, pipeconf),
- ACT_GRP_STR, MODIFY_STR)) {
- // recover group information
- completeFuture(client.writeActionGroup(groupOnDevice, MODIFY, pipeconf),
- ACT_GRP_STR, MODIFY_STR);
- // remove what we added
- completeFuture(client.writeActionGroupMembers(groupProfileId, membersToAdd, DELETE, pipeconf),
- ACT_GRP_MEMS_STR, INSERT_STR);
- return false;
- }
-
- if (!membersToRemove.isEmpty() &&
- !completeFuture(client.writeActionGroupMembers(groupProfileId, membersToRemove, DELETE, pipeconf),
- ACT_GRP_MEMS_STR, DELETE_STR)) {
- // add what we removed
- completeFuture(client.writeActionGroupMembers(groupProfileId, membersToRemove, INSERT, pipeconf),
- ACT_GRP_MEMS_STR, DELETE_STR);
- // recover group information
- completeFuture(client.writeActionGroup(groupOnDevice, MODIFY, pipeconf),
- ACT_GRP_STR, MODIFY_STR);
- // remove what we added
- completeFuture(client.writeActionGroupMembers(groupProfileId, membersToAdd, DELETE, pipeconf),
- ACT_GRP_MEMS_STR, INSERT_STR);
- return false;
- }
-
- return true;
- }
-
- private boolean writeGroupToDevice(PiActionGroup groupToApply) {
- // First insert members, then group.
- // The operation is deemed successful if both operations are successful.
- // FIXME: add transactional semantics, i.e. remove members if group fails.
- final boolean membersSuccess = completeFuture(
- client.writeActionGroupMembers(groupToApply.actionProfileId(),
- groupToApply.members(),
- INSERT, pipeconf),
- ACT_GRP_MEMS_STR, INSERT_STR);
- return membersSuccess && completeFuture(
- client.writeActionGroup(groupToApply, INSERT, pipeconf),
- ACT_GRP_STR, INSERT_STR);
- }
-
- private boolean deleteGroupFromDevice(PiActionGroup piActionGroup) {
- // First delete group, then members.
- // The operation is deemed successful if both operations are successful.
- final boolean groupSuccess = completeFuture(
- client.writeActionGroup(piActionGroup, DELETE, pipeconf),
- ACT_GRP_STR, DELETE_STR);
- return groupSuccess && completeFuture(
- client.writeActionGroupMembers(piActionGroup.actionProfileId(),
- piActionGroup.members(),
- DELETE, pipeconf),
- ACT_GRP_MEMS_STR, DELETE_STR);
- }
-
- private boolean completeFuture(CompletableFuture<Boolean> completableFuture,
- String topic, String action) {
- return getFutureWithDeadline(
- completableFuture, format("performing %s %s", action, topic), false);
- }
-
- private Stream<PiActionGroup> streamPiGroupsFromDevice(PiActionProfileId actProfId) {
- // Read PI groups and return original PD one.
- // TODO: implement P4Runtime client call to read all groups with one call
- // Good is pipeline has multiple action profiles.
- final Collection<PiActionGroup> groups = getFutureWithDeadline(
- client.dumpGroups(actProfId, pipeconf),
- "dumping groups", Collections.emptyList());
- return groups.stream();
- }
-
- private Group forgeGroupEntry(PiActionGroup piGroup) {
- final PiActionGroupHandle handle = PiActionGroupHandle.of(deviceId, piGroup);
- final Optional<PiTranslatedEntity<Group, PiActionGroup>>
- translatedEntity = groupTranslator.lookup(handle);
- final TimedEntry<PiActionGroup> timedEntry = groupMirror.get(handle);
- // Is entry consistent with our state?
- if (!translatedEntity.isPresent()) {
- log.warn("Group handle not found in translation store: {}", handle);
- return null;
- }
- if (timedEntry == null) {
- // Don't bother logging more than debug, most probably it's the EC
- // map backing the store that has not received all the updates yet.
- log.debug("Group handle not found in device mirror: {}", handle);
- return null;
- }
- return addedGroup(translatedEntity.get().original(), timedEntry.lifeSec());
- }
-
- private Group forgeMcGroupEntry(PiMulticastGroupEntry mcGroup) {
- final PiMulticastGroupEntryHandle handle = PiMulticastGroupEntryHandle.of(
- deviceId, mcGroup);
- final Optional<PiTranslatedEntity<Group, PiMulticastGroupEntry>>
- translatedEntity = mcGroupTranslator.lookup(handle);
- final TimedEntry<PiMulticastGroupEntry> timedEntry = mcGroupMirror.get(handle);
- // Is entry consistent with our state?
- if (!translatedEntity.isPresent()) {
- log.warn("Multicast group handle not found in translation store: {}", handle);
- return null;
- }
- if (timedEntry == null) {
- log.warn("Multicast group handle not found in device mirror: {}", handle);
- return null;
- }
- return addedGroup(translatedEntity.get().original(), timedEntry.lifeSec());
- }
-
- private Group addedGroup(Group original, long life) {
- final DefaultGroup forgedGroup = new DefaultGroup(original.id(), original);
- forgedGroup.setState(Group.GroupState.ADDED);
- forgedGroup.setLife(life);
- return forgedGroup;
}
}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeMulticastGroupProgrammable.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeMulticastGroupProgrammable.java
new file mode 100644
index 0000000..a5c3538
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeMulticastGroupProgrammable.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Striped;
+import org.onosproject.drivers.p4runtime.mirror.P4RuntimeMulticastGroupMirror;
+import org.onosproject.drivers.p4runtime.mirror.TimedEntry;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.group.DefaultGroup;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupDescription;
+import org.onosproject.net.group.GroupOperation;
+import org.onosproject.net.group.GroupOperations;
+import org.onosproject.net.group.GroupProgrammable;
+import org.onosproject.net.group.GroupStore;
+import org.onosproject.net.pi.runtime.PiMulticastGroupEntry;
+import org.onosproject.net.pi.runtime.PiMulticastGroupEntryHandle;
+import org.onosproject.net.pi.service.PiMulticastGroupTranslator;
+import org.onosproject.net.pi.service.PiTranslatedEntity;
+import org.onosproject.net.pi.service.PiTranslationException;
+import org.onosproject.p4runtime.api.P4RuntimeClient;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.locks.Lock;
+import java.util.stream.Collectors;
+
+import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.DELETE;
+import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.INSERT;
+import static org.onosproject.p4runtime.api.P4RuntimeClient.WriteOperationType.MODIFY;
+
+/**
+ * Implementation of GroupProgrammable to handle multicast groups in P4Runtime.
+ */
+public class P4RuntimeMulticastGroupProgrammable
+ extends AbstractP4RuntimeHandlerBehaviour implements GroupProgrammable {
+
+ // Needed to synchronize operations over the same group.
+ private static final Striped<Lock> STRIPED_LOCKS = Striped.lock(30);
+
+ private GroupStore groupStore;
+ private P4RuntimeMulticastGroupMirror mcGroupMirror;
+ private PiMulticastGroupTranslator mcGroupTranslator;
+
+ @Override
+ protected boolean setupBehaviour() {
+ if (!super.setupBehaviour()) {
+ return false;
+ }
+ mcGroupMirror = this.handler().get(P4RuntimeMulticastGroupMirror.class);
+ groupStore = handler().get(GroupStore.class);
+ mcGroupTranslator = piTranslationService.multicastGroupTranslator();
+ return true;
+ }
+
+ @Override
+ public void performGroupOperation(DeviceId deviceId, GroupOperations groupOps) {
+ if (!setupBehaviour()) {
+ return;
+ }
+ groupOps.operations().stream()
+ .filter(op -> op.groupType().equals(GroupDescription.Type.ALL))
+ .forEach(op -> {
+ final Group group = groupStore.getGroup(deviceId, op.groupId());
+ processMcGroupOp(group, op.opType());
+ });
+ }
+
+ @Override
+ public Collection<Group> getGroups() {
+ if (!setupBehaviour()) {
+ return Collections.emptyList();
+ }
+ return ImmutableList.copyOf(getMcGroups());
+ }
+
+ private Collection<Group> getMcGroups() {
+ // TODO: missing support for reading multicast groups is ready in PI/Stratum.
+ return getMcGroupsFromMirror();
+ }
+
+ private Collection<Group> getMcGroupsFromMirror() {
+ return mcGroupMirror.getAll(deviceId).stream()
+ .map(TimedEntry::entry)
+ .map(this::forgeMcGroupEntry)
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+ }
+
+ private void processMcGroupOp(Group pdGroup, GroupOperation.Type opType) {
+ final PiMulticastGroupEntry mcGroup;
+ try {
+ mcGroup = mcGroupTranslator.translate(pdGroup, pipeconf);
+ } catch (PiTranslationException e) {
+ log.warn("Unable to translate multicast group, aborting {} operation: {} [{}]",
+ opType, e.getMessage(), pdGroup);
+ return;
+ }
+ final PiMulticastGroupEntryHandle handle = PiMulticastGroupEntryHandle.of(
+ deviceId, mcGroup);
+ final PiMulticastGroupEntry groupOnDevice = mcGroupMirror.get(handle) == null
+ ? null
+ : mcGroupMirror.get(handle).entry();
+ final Lock lock = STRIPED_LOCKS.get(handle);
+ lock.lock();
+ try {
+ processMcGroup(handle, mcGroup,
+ groupOnDevice, pdGroup, opType);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private void processMcGroup(PiMulticastGroupEntryHandle handle,
+ PiMulticastGroupEntry groupToApply,
+ PiMulticastGroupEntry groupOnDevice,
+ Group pdGroup, GroupOperation.Type opType) {
+ switch (opType) {
+ case ADD:
+ robustMcGroupAdd(handle, groupToApply, pdGroup);
+ return;
+ case MODIFY:
+ // Since reading multicast groups is not supported yet on
+ // PI/Stratum, we cannot trust groupOnDevic) as we don't have a
+ // mechanism to enforce consistency of the mirror with the
+ // device state.
+ // if (driverBoolProperty(CHECK_MIRROR_BEFORE_UPDATE,
+ // DEFAULT_CHECK_MIRROR_BEFORE_UPDATE)
+ // && p4OpType == MODIFY
+ // && groupOnDevice != null
+ // && groupOnDevice.equals(groupToApply)) {
+ // // Ignore.
+ // return;
+ // }
+ robustMcGroupModify(handle, groupToApply, pdGroup);
+ return;
+ case DELETE:
+ mcGroupApply(handle, groupToApply, pdGroup, DELETE);
+ return;
+ default:
+ log.error("Unknown group operation type {}, " +
+ "cannot process multicast group", opType);
+ }
+ }
+
+ private boolean writeMcGroupOnDevice(PiMulticastGroupEntry group, P4RuntimeClient.WriteOperationType opType) {
+ return getFutureWithDeadline(
+ client.writePreMulticastGroupEntries(
+ Collections.singletonList(group), opType),
+ "performing multicast group " + opType, false);
+ }
+
+ private boolean mcGroupApply(PiMulticastGroupEntryHandle handle,
+ PiMulticastGroupEntry piGroup,
+ Group pdGroup,
+ P4RuntimeClient.WriteOperationType opType) {
+ switch (opType) {
+ case DELETE:
+ if (writeMcGroupOnDevice(piGroup, DELETE)) {
+ mcGroupMirror.remove(handle);
+ mcGroupTranslator.forget(handle);
+ return true;
+ } else {
+ return false;
+ }
+ case INSERT:
+ case MODIFY:
+ if (writeMcGroupOnDevice(piGroup, opType)) {
+ mcGroupMirror.put(handle, piGroup);
+ mcGroupTranslator.learn(handle, new PiTranslatedEntity<>(
+ pdGroup, piGroup, handle));
+ return true;
+ } else {
+ return false;
+ }
+ default:
+ log.warn("Unknown operation type {}, cannot apply group", opType);
+ return false;
+ }
+ }
+
+ private void robustMcGroupAdd(PiMulticastGroupEntryHandle handle,
+ PiMulticastGroupEntry piGroup,
+ Group pdGroup) {
+ if (mcGroupApply(handle, piGroup, pdGroup, INSERT)) {
+ return;
+ }
+ // Try to delete (perhaps it already exists) and re-add...
+ mcGroupApply(handle, piGroup, pdGroup, DELETE);
+ mcGroupApply(handle, piGroup, pdGroup, INSERT);
+ }
+
+ private void robustMcGroupModify(PiMulticastGroupEntryHandle handle,
+ PiMulticastGroupEntry piGroup,
+ Group pdGroup) {
+ if (mcGroupApply(handle, piGroup, pdGroup, MODIFY)) {
+ return;
+ }
+ // Not sure for which reason it cannot be modified, so try to delete and insert instead...
+ mcGroupApply(handle, piGroup, pdGroup, DELETE);
+ mcGroupApply(handle, piGroup, pdGroup, INSERT);
+ }
+
+ private Group forgeMcGroupEntry(PiMulticastGroupEntry mcGroup) {
+ final PiMulticastGroupEntryHandle handle = PiMulticastGroupEntryHandle.of(
+ deviceId, mcGroup);
+ final Optional<PiTranslatedEntity<Group, PiMulticastGroupEntry>>
+ translatedEntity = mcGroupTranslator.lookup(handle);
+ final TimedEntry<PiMulticastGroupEntry> timedEntry = mcGroupMirror.get(handle);
+ // Is entry consistent with our state?
+ if (!translatedEntity.isPresent()) {
+ log.warn("Multicast group handle not found in translation store: {}", handle);
+ return null;
+ }
+ if (timedEntry == null) {
+ log.warn("Multicast group handle not found in device mirror: {}", handle);
+ return null;
+ }
+ return addedGroup(translatedEntity.get().original(), timedEntry.lifeSec());
+ }
+
+ private Group addedGroup(Group original, long life) {
+ final DefaultGroup forgedGroup = new DefaultGroup(original.id(), original);
+ forgedGroup.setState(Group.GroupState.ADDED);
+ forgedGroup.setLife(life);
+ return forgedGroup;
+ }
+
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/AbstractDistributedP4RuntimeMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/AbstractDistributedP4RuntimeMirror.java
index c5eb4c0..9230547 100644
--- a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/AbstractDistributedP4RuntimeMirror.java
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/AbstractDistributedP4RuntimeMirror.java
@@ -17,6 +17,7 @@
package org.onosproject.drivers.p4runtime.mirror;
import com.google.common.annotations.Beta;
+import com.google.common.collect.Maps;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
@@ -136,41 +137,54 @@
@Override
public void sync(DeviceId deviceId, Map<H, E> deviceState) {
checkNotNull(deviceId);
- Set<Map.Entry<H, TimedEntry<E>>> localEntries = getEntriesForDevice(deviceId);
+ final Map<H, E> localState = getMirrorMapForDevice(deviceId);
+
final AtomicInteger removeCount = new AtomicInteger(0);
final AtomicInteger updateCount = new AtomicInteger(0);
- localEntries.forEach(e -> {
- final H handle = e.getKey();
- final E storedValue = e.getValue().entry();
- if (!deviceState.containsKey(handle)) {
- log.debug("Removing mirror entry for {}: {}", deviceId, storedValue);
+ final AtomicInteger addCount = new AtomicInteger(0);
+ // Add missing entries.
+ deviceState.keySet().stream()
+ .filter(deviceHandle -> !localState.containsKey(deviceHandle))
+ .forEach(deviceHandle -> {
+ final E entryToAdd = deviceState.get(deviceHandle);
+ log.debug("Adding mirror entry for {}: {}",
+ deviceId, entryToAdd);
+ put(deviceHandle, entryToAdd);
+ addCount.incrementAndGet();
+ });
+ // Update or remove local entries.
+ localState.keySet().forEach(localHandle -> {
+ final E localEntry = localState.get(localHandle);
+ final E deviceEntry = deviceState.get(localHandle);
+ if (deviceEntry == null) {
+ log.debug("Removing mirror entry for {}: {}", deviceId, localEntry);
+ remove(localHandle);
removeCount.incrementAndGet();
- } else {
- final E deviceValue = deviceState.get(handle);
- if (!deviceValue.equals(storedValue)) {
- log.debug("Updating mirror entry for {}: {}-->{}",
- deviceId, storedValue, deviceValue);
- put(handle, deviceValue);
- updateCount.incrementAndGet();
- }
+ } else if (!deviceEntry.equals(localEntry)) {
+ log.debug("Updating mirror entry for {}: {}-->{}",
+ deviceId, localEntry, deviceEntry);
+ put(localHandle, deviceEntry);
+ updateCount.incrementAndGet();
}
});
- if (removeCount.get() + updateCount.get() > 0) {
- log.info("Synchronized mirror entries for {}: {} removed, {} updated",
- deviceId, removeCount, updateCount);
+ if (removeCount.get() + updateCount.get() + addCount.get() > 0) {
+ log.info("Synchronized mirror entries for {}: {} removed, {} updated, {} added",
+ deviceId, removeCount, updateCount, addCount);
}
}
- private Collection<H> getHandlesForDevice(DeviceId deviceId) {
+ private Set<H> getHandlesForDevice(DeviceId deviceId) {
return mirrorMap.keySet().stream()
.filter(h -> h.deviceId().equals(deviceId))
- .collect(Collectors.toList());
+ .collect(Collectors.toSet());
}
- private Set<Map.Entry<H, TimedEntry<E>>> getEntriesForDevice(DeviceId deviceId) {
- return mirrorMap.entrySet().stream()
+ private Map<H, E> getMirrorMapForDevice(DeviceId deviceId) {
+ final Map<H, E> deviceMap = Maps.newHashMap();
+ mirrorMap.entrySet().stream()
.filter(e -> e.getKey().deviceId().equals(deviceId))
- .collect(Collectors.toSet());
+ .forEach(e -> deviceMap.put(e.getKey(), e.getValue().entry()));
+ return deviceMap;
}
private void removeAll(DeviceId deviceId) {
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeActionProfileMemberMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeActionProfileMemberMirror.java
new file mode 100644
index 0000000..83edba4
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeActionProfileMemberMirror.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.net.pi.runtime.PiActionGroupMember;
+import org.onosproject.net.pi.runtime.PiActionGroupMemberHandle;
+import org.onosproject.store.serializers.KryoNamespaces;
+
+/**
+ * Distributed implementation of a P4Runtime action profile member mirror.
+ */
+@Component(immediate = true)
+@Service
+public class DistributedP4RuntimeActionProfileMemberMirror
+ extends AbstractDistributedP4RuntimeMirror
+ <PiActionGroupMemberHandle, PiActionGroupMember>
+ implements P4RuntimeActionProfileMemberMirror {
+
+ private static final String DIST_MAP_NAME = "onos-p4runtime-act-prof-member-mirror";
+
+ @Override
+ String mapName() {
+ return DIST_MAP_NAME;
+ }
+
+ @Override
+ KryoNamespace storeSerializer() {
+ return KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(TimedEntry.class)
+ .build();
+ }
+}
diff --git a/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeActionProfileMemberMirror.java b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeActionProfileMemberMirror.java
new file mode 100644
index 0000000..8ab1fa0
--- /dev/null
+++ b/drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/P4RuntimeActionProfileMemberMirror.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import org.onosproject.net.pi.runtime.PiActionGroupMember;
+import org.onosproject.net.pi.runtime.PiActionGroupMemberHandle;
+
+/**
+ * Mirror of action profile members installed on a P4Runtime device.
+ */
+public interface P4RuntimeActionProfileMemberMirror
+ extends P4RuntimeMirror<PiActionGroupMemberHandle, PiActionGroupMember> {
+}