blob: fca05cbb2deba27394ee8ece1320894590024119 [file] [log] [blame]
Carmelo Casconee44592f2018-09-12 02:24:47 -07001/*
2 * Copyright 2017-present Open Networking Foundation
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package org.onosproject.drivers.p4runtime;
18
Carmelo Casconee44592f2018-09-12 02:24:47 -070019import com.google.common.collect.Lists;
Carmelo Casconee44592f2018-09-12 02:24:47 -070020import com.google.common.collect.Sets;
Carmelo Casconee44592f2018-09-12 02:24:47 -070021import org.onlab.util.SharedExecutors;
Carmelo Casconecb4327a2018-09-11 15:17:23 -070022import org.onosproject.drivers.p4runtime.mirror.P4RuntimeActionProfileGroupMirror;
Carmelo Casconee44592f2018-09-12 02:24:47 -070023import org.onosproject.drivers.p4runtime.mirror.P4RuntimeActionProfileMemberMirror;
Carmelo Cascone4c289b72019-01-22 15:30:45 -080024import org.onosproject.drivers.p4runtime.mirror.P4RuntimeMirror;
Carmelo Casconee44592f2018-09-12 02:24:47 -070025import org.onosproject.drivers.p4runtime.mirror.TimedEntry;
26import org.onosproject.net.DeviceId;
27import org.onosproject.net.group.DefaultGroup;
28import org.onosproject.net.group.DefaultGroupDescription;
29import org.onosproject.net.group.Group;
30import org.onosproject.net.group.GroupDescription;
31import org.onosproject.net.group.GroupOperation;
32import org.onosproject.net.group.GroupOperations;
33import org.onosproject.net.group.GroupProgrammable;
34import org.onosproject.net.group.GroupStore;
Carmelo Casconee44592f2018-09-12 02:24:47 -070035import org.onosproject.net.pi.model.PiActionProfileModel;
Carmelo Casconecb4327a2018-09-11 15:17:23 -070036import org.onosproject.net.pi.runtime.PiActionProfileGroup;
37import org.onosproject.net.pi.runtime.PiActionProfileGroupHandle;
38import org.onosproject.net.pi.runtime.PiActionProfileMember;
39import org.onosproject.net.pi.runtime.PiActionProfileMemberHandle;
Carmelo Cascone4c289b72019-01-22 15:30:45 -080040import org.onosproject.net.pi.runtime.PiEntity;
41import org.onosproject.net.pi.runtime.PiHandle;
Carmelo Casconee44592f2018-09-12 02:24:47 -070042import org.onosproject.net.pi.service.PiGroupTranslator;
43import org.onosproject.net.pi.service.PiTranslatedEntity;
44import org.onosproject.net.pi.service.PiTranslationException;
Carmelo Cascone4c289b72019-01-22 15:30:45 -080045import org.onosproject.p4runtime.api.P4RuntimeReadClient;
46import org.onosproject.p4runtime.api.P4RuntimeWriteClient;
Carmelo Casconee44592f2018-09-12 02:24:47 -070047
48import java.util.Collection;
49import java.util.Collections;
50import java.util.List;
51import java.util.Map;
52import java.util.Objects;
53import java.util.Optional;
54import java.util.Set;
Carmelo Casconee44592f2018-09-12 02:24:47 -070055import java.util.stream.Collectors;
Carmelo Casconee44592f2018-09-12 02:24:47 -070056
Carmelo Cascone99c59db2019-01-17 15:39:35 -080057import static java.util.stream.Collectors.toMap;
Carmelo Cascone4c289b72019-01-22 15:30:45 -080058import static java.util.stream.Collectors.toSet;
Carmelo Casconee44592f2018-09-12 02:24:47 -070059
60/**
61 * Implementation of GroupProgrammable to handle action profile groups in
62 * P4Runtime.
63 */
64public class P4RuntimeActionGroupProgrammable
65 extends AbstractP4RuntimeHandlerBehaviour
66 implements GroupProgrammable {
67
68 // If true, we avoid querying the device and return what's already known by
69 // the ONOS store.
70 private static final String READ_ACTION_GROUPS_FROM_MIRROR = "actionGroupReadFromMirror";
71 private static final boolean DEFAULT_READ_ACTION_GROUPS_FROM_MIRROR = false;
72
73 protected GroupStore groupStore;
Carmelo Casconecb4327a2018-09-11 15:17:23 -070074 private P4RuntimeActionProfileGroupMirror groupMirror;
Carmelo Casconee44592f2018-09-12 02:24:47 -070075 private P4RuntimeActionProfileMemberMirror memberMirror;
76 private PiGroupTranslator groupTranslator;
77
Carmelo Casconee44592f2018-09-12 02:24:47 -070078 @Override
79 protected boolean setupBehaviour() {
80 if (!super.setupBehaviour()) {
81 return false;
82 }
Carmelo Casconecb4327a2018-09-11 15:17:23 -070083 groupMirror = this.handler().get(P4RuntimeActionProfileGroupMirror.class);
Carmelo Casconee44592f2018-09-12 02:24:47 -070084 memberMirror = this.handler().get(P4RuntimeActionProfileMemberMirror.class);
85 groupStore = handler().get(GroupStore.class);
Yi Tsengd7716482018-10-31 15:34:30 -070086 groupTranslator = translationService.groupTranslator();
Carmelo Casconee44592f2018-09-12 02:24:47 -070087 return true;
88 }
89
90 @Override
91 public void performGroupOperation(DeviceId deviceId,
92 GroupOperations groupOps) {
93 if (!setupBehaviour()) {
94 return;
95 }
96
97 groupOps.operations().stream()
98 .filter(op -> !op.groupType().equals(GroupDescription.Type.ALL))
99 .forEach(op -> {
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800100 // ONOS-7785 We need the group app cookie (which includes
101 // the action profile ID) but this is not part of the
102 // GroupDescription.
Carmelo Casconee44592f2018-09-12 02:24:47 -0700103 Group groupOnStore = groupStore.getGroup(deviceId, op.groupId());
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800104 if (groupOnStore == null) {
105 log.warn("Unable to find group {} in store, aborting {} operation",
106 op.groupId(), op.opType());
107 return;
108 }
Carmelo Casconee44592f2018-09-12 02:24:47 -0700109 GroupDescription groupDesc = new DefaultGroupDescription(
110 deviceId, op.groupType(), op.buckets(), groupOnStore.appCookie(),
111 op.groupId().id(), groupOnStore.appId());
112 DefaultGroup groupToApply = new DefaultGroup(op.groupId(), groupDesc);
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800113 processPdGroup(groupToApply, op.opType());
Carmelo Casconee44592f2018-09-12 02:24:47 -0700114 });
115 }
116
117 @Override
118 public Collection<Group> getGroups() {
119 if (!setupBehaviour()) {
120 return Collections.emptyList();
121 }
Carmelo Casconee44592f2018-09-12 02:24:47 -0700122
123 if (driverBoolProperty(READ_ACTION_GROUPS_FROM_MIRROR,
124 DEFAULT_READ_ACTION_GROUPS_FROM_MIRROR)) {
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800125 return getGroupsFromMirror();
Carmelo Casconee44592f2018-09-12 02:24:47 -0700126 }
127
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800128 // Dump groups and members from device for all action profiles.
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800129 final P4RuntimeReadClient.ReadRequest request = client.read(pipeconf);
130 pipeconf.pipelineModel().actionProfiles()
131 .stream().map(PiActionProfileModel::id)
132 .forEach(id -> request.actionProfileGroups(id)
133 .actionProfileMembers(id));
134 final P4RuntimeReadClient.ReadResponse response = request.submitSync();
135
136 if (!response.isSuccess()) {
137 // Error at client level.
138 return Collections.emptyList();
139 }
140
141 final Collection<PiActionProfileGroup> groupsOnDevice = response.all(
142 PiActionProfileGroup.class);
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800143 final Map<PiActionProfileMemberHandle, PiActionProfileMember> membersOnDevice =
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800144 response.all(PiActionProfileMember.class).stream()
145 .collect(toMap(m -> m.handle(deviceId), m -> m));
Carmelo Casconee44592f2018-09-12 02:24:47 -0700146
147 // Sync mirrors.
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800148 groupMirror.sync(deviceId, groupsOnDevice);
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800149 memberMirror.sync(deviceId, membersOnDevice.values());
Carmelo Casconee44592f2018-09-12 02:24:47 -0700150
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800151 // Retrieve the original PD group before translation.
Carmelo Casconee44592f2018-09-12 02:24:47 -0700152 final List<Group> result = Lists.newArrayList();
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800153 final List<PiActionProfileGroup> groupsToRemove = Lists.newArrayList();
154 final Set<PiActionProfileMemberHandle> memberHandlesToKeep = Sets.newHashSet();
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800155 for (PiActionProfileGroup piGroup : groupsOnDevice) {
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800156 final Group pdGroup = checkAndForgeGroupEntry(piGroup, membersOnDevice);
Carmelo Casconee44592f2018-09-12 02:24:47 -0700157 if (pdGroup == null) {
158 // Entry is on device but unknown to translation service or
159 // device mirror. Inconsistent. Mark for removal.
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800160 groupsToRemove.add(piGroup);
Carmelo Casconee44592f2018-09-12 02:24:47 -0700161 } else {
Carmelo Casconee44592f2018-09-12 02:24:47 -0700162 result.add(pdGroup);
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800163 // Keep track of member handles used in groups.
164 piGroup.members().stream()
165 .map(m -> PiActionProfileMemberHandle.of(
166 deviceId, piGroup.actionProfile(), m.id()))
167 .forEach(memberHandlesToKeep::add);
Carmelo Casconee44592f2018-09-12 02:24:47 -0700168 }
169 }
170
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800171 // Trigger clean up of inconsistent groups and members (if any). This
172 // process takes care of removing any orphan member, e.g. from a
173 // partial/unsuccessful group insertion.
174 // This will update the mirror accordingly.
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800175 final Set<PiActionProfileMemberHandle> memberHandlesToRemove = Sets.difference(
176 membersOnDevice.keySet(), memberHandlesToKeep);
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800177 final Set<PiActionProfileGroupHandle> groupHandlesToRemove = groupsToRemove
178 .stream().map(g -> g.handle(deviceId)).collect(toSet());
179 if (groupHandlesToRemove.size() + memberHandlesToRemove.size() > 0) {
180 log.warn("Cleaning up {} action profile groups and " +
181 "{} members on {}...",
182 groupHandlesToRemove.size(), memberHandlesToRemove.size(), deviceId);
183 SharedExecutors.getSingleThreadExecutor().execute(
184 () -> submitWriteRequestAndUpdateMirror(
185 client.write(pipeconf)
186 .delete(groupHandlesToRemove)
187 .delete(memberHandlesToRemove)));
188 }
Carmelo Casconee44592f2018-09-12 02:24:47 -0700189
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800190 // Done.
Carmelo Casconee44592f2018-09-12 02:24:47 -0700191 return result;
192 }
193
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800194 private Collection<Group> getGroupsFromMirror() {
195 final Map<PiActionProfileMemberHandle, PiActionProfileMember> members =
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800196 memberMirror.getAll(deviceId).stream()
197 .map(TimedEntry::entry)
198 .collect(toMap(e -> e.handle(deviceId), e -> e));
Carmelo Casconee44592f2018-09-12 02:24:47 -0700199 return groupMirror.getAll(deviceId).stream()
200 .map(TimedEntry::entry)
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800201 .map(g -> checkAndForgeGroupEntry(
202 g, members))
Carmelo Casconee44592f2018-09-12 02:24:47 -0700203 .filter(Objects::nonNull)
204 .collect(Collectors.toList());
205 }
206
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800207 private Group checkAndForgeGroupEntry(
208 PiActionProfileGroup piGroupOnDevice,
209 Map<PiActionProfileMemberHandle, PiActionProfileMember> membersOnDevice) {
210 final PiActionProfileGroupHandle handle = PiActionProfileGroupHandle.of(
211 deviceId, piGroupOnDevice);
Carmelo Casconecb4327a2018-09-11 15:17:23 -0700212 final Optional<PiTranslatedEntity<Group, PiActionProfileGroup>>
Carmelo Casconee44592f2018-09-12 02:24:47 -0700213 translatedEntity = groupTranslator.lookup(handle);
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800214 final TimedEntry<PiActionProfileGroup> mirrorEntry = groupMirror.get(handle);
215 // Check that entry obtained from device is consistent with what is known
216 // by the translation store.
Carmelo Casconee44592f2018-09-12 02:24:47 -0700217 if (!translatedEntity.isPresent()) {
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800218 log.warn("Group not found in translation store: {}", handle);
Carmelo Casconee44592f2018-09-12 02:24:47 -0700219 return null;
220 }
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800221 final PiActionProfileGroup piGroupFromStore = translatedEntity.get().translated();
222 if (!piGroupFromStore.equals(piGroupOnDevice)) {
223 log.warn("Group on device {} is different from the one in " +
224 "translation store: {} [device={}, store={}]",
225 deviceId, handle, piGroupOnDevice, piGroupFromStore);
Carmelo Casconee44592f2018-09-12 02:24:47 -0700226 return null;
227 }
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800228 // Groups in P4Runtime contains only a reference to members. Check that
229 // the actual member instances in the translation store are the same
230 // found on the device.
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800231 if (!validateGroupMembers(piGroupFromStore, membersOnDevice)) {
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800232 log.warn("Group on device {} refers to members that are different " +
233 "than those found in translation store: {}", handle);
234 return null;
235 }
236 if (mirrorEntry == null) {
Carmelo Casconee44592f2018-09-12 02:24:47 -0700237 log.warn("Group handle not found in device mirror: {}", handle);
238 return null;
239 }
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800240 // Check that members from device are the same as in the translated group.
241 return addedGroup(translatedEntity.get().original(), mirrorEntry.lifeSec());
242 }
243
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800244 private boolean validateGroupMembers(
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800245 PiActionProfileGroup piGroupFromStore,
246 Map<PiActionProfileMemberHandle, PiActionProfileMember> membersOnDevice) {
247 final Collection<PiActionProfileMember> groupMembers =
248 extractAllMemberInstancesOrNull(piGroupFromStore);
249 if (groupMembers == null) {
250 return false;
251 }
252 return groupMembers.stream().allMatch(
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800253 memberFromStore -> memberFromStore.equals(membersOnDevice.get(
254 memberFromStore.handle(deviceId))));
Carmelo Casconee44592f2018-09-12 02:24:47 -0700255 }
256
257 private Group addedGroup(Group original, long life) {
258 final DefaultGroup forgedGroup = new DefaultGroup(original.id(), original);
259 forgedGroup.setState(Group.GroupState.ADDED);
260 forgedGroup.setLife(life);
261 return forgedGroup;
262 }
263
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800264 private void processPdGroup(Group pdGroup, GroupOperation.Type opType) {
Carmelo Casconecb4327a2018-09-11 15:17:23 -0700265 final PiActionProfileGroup piGroup;
Carmelo Casconee44592f2018-09-12 02:24:47 -0700266 try {
267 piGroup = groupTranslator.translate(pdGroup, pipeconf);
268 } catch (PiTranslationException e) {
269 log.warn("Unable to translate group, aborting {} operation: {} [{}]",
270 opType, e.getMessage(), pdGroup);
271 return;
272 }
273 final Operation operation = opType.equals(GroupOperation.Type.DELETE)
274 ? Operation.REMOVE : Operation.APPLY;
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800275 final PiActionProfileGroupHandle handle = piGroup.handle(deviceId);
276 if (writePiGroupOnDevice(piGroup, handle, operation)) {
277 if (operation.equals(Operation.APPLY)) {
278 groupTranslator.learn(handle, new PiTranslatedEntity<>(
279 pdGroup, piGroup, handle));
280 } else {
281 groupTranslator.forget(handle);
Carmelo Casconee44592f2018-09-12 02:24:47 -0700282 }
Carmelo Casconee44592f2018-09-12 02:24:47 -0700283 }
284 }
285
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800286 private boolean writePiGroupOnDevice(
287 PiActionProfileGroup group,
288 PiActionProfileGroupHandle groupHandle,
289 Operation operation) {
290 // Generate a write request to write both members and groups. Return
291 // true if request is successful or if there's no need to write on
292 // device (according to mirror state), otherwise, return false.
293 final Collection<PiActionProfileMember> members = extractAllMemberInstancesOrNull(group);
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800294 if (members == null) {
295 return false;
296 }
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800297 final P4RuntimeWriteClient.WriteRequest request = client.write(pipeconf);
298 // FIXME: when operation is remove, should we remove members first? Same
299 // thing when modifying a group, should we first modify the group then
300 // remove the member?
301 final boolean allMembersSkipped = members.stream()
302 .allMatch(m -> appendEntityToWriteRequestOrSkip(
303 request, m.handle(deviceId), m, memberMirror, operation));
304 final boolean groupSkipped = appendEntityToWriteRequestOrSkip(
305 request, groupHandle, group, groupMirror, operation);
306 if (allMembersSkipped && groupSkipped) {
Carmelo Casconee44592f2018-09-12 02:24:47 -0700307 return true;
308 } else {
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800309 // True if all entities in the request (groups and members) where
310 // written successfully.
311 return submitWriteRequestAndUpdateMirror(request).isSuccess();
Carmelo Casconee44592f2018-09-12 02:24:47 -0700312 }
313 }
314
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800315 private <H extends PiHandle, E extends PiEntity> boolean appendEntityToWriteRequestOrSkip(
316 P4RuntimeWriteClient.WriteRequest writeRequest,
317 H handle,
318 E entityToApply,
319 P4RuntimeMirror<H, E> mirror,
320 Operation operation) {
321 // Should return true if there's no need to write entity on device,
322 // false if the write request is modified or an error occurs.
323 final TimedEntry<E> entityOnDevice = mirror.get(handle);
324 switch (operation) {
325 case APPLY:
326 if (entityOnDevice == null) {
327 writeRequest.insert(entityToApply);
328 } else if (entityToApply.equals(entityOnDevice.entry())) {
329 // Skip writing if group is unchanged.
330 return true;
331 } else {
332 writeRequest.modify(entityToApply);
333 }
334 break;
335 case REMOVE:
336 if (entityOnDevice == null) {
337 // Skip deleting if group does not exist on device.
338 return true;
339 } else {
340 writeRequest.delete(handle);
341 }
342 break;
343 default:
344 log.error("Unrecognized operation {}", operation);
345 break;
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800346 }
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800347 return false;
Carmelo Cascone99c59db2019-01-17 15:39:35 -0800348 }
349
350 private Collection<PiActionProfileMember> extractAllMemberInstancesOrNull(
351 PiActionProfileGroup group) {
352 final Collection<PiActionProfileMember> instances = group.members().stream()
353 .map(PiActionProfileGroup.WeightedMember::instance)
354 .filter(Objects::nonNull)
355 .collect(Collectors.toList());
356 if (instances.size() != group.members().size()) {
357 log.error("PiActionProfileGroup has {} member references, " +
358 "but only {} instances were found",
359 group.members().size(), instances.size());
360 return null;
361 }
362 return instances;
Carmelo Casconee44592f2018-09-12 02:24:47 -0700363 }
364
Carmelo Cascone4c289b72019-01-22 15:30:45 -0800365 private P4RuntimeWriteClient.WriteResponse submitWriteRequestAndUpdateMirror(
366 P4RuntimeWriteClient.WriteRequest request) {
367 final P4RuntimeWriteClient.WriteResponse response = request.submitSync();
368 groupMirror.replayWriteResponse(response);
369 memberMirror.replayWriteResponse(response);
370 return response;
371 }
372
Carmelo Casconee44592f2018-09-12 02:24:47 -0700373 enum Operation {
374 APPLY, REMOVE
375 }
376}