[CORD-1108] Refactoring OFDPA Group Handler
Create package "ofdpa" under pipeline package
Move helper functions and classes to OfdpaGroupHandlerUtility
Change-Id: I47e42f2c8afc9088ed684cd6a087233a82c452f6
diff --git a/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2GroupHandler.java b/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2GroupHandler.java
new file mode 100644
index 0000000..aaca495
--- /dev/null
+++ b/drivers/default/src/main/java/org/onosproject/driver/pipeline/ofdpa/Ofdpa2GroupHandler.java
@@ -0,0 +1,1641 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.driver.pipeline.ofdpa;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalCause;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.onlab.osgi.ServiceDirectory;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.MplsLabel;
+import org.onlab.packet.VlanId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.GroupId;
+import org.onosproject.driver.extensions.OfdpaSetVlanVid;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.behaviour.NextGroup;
+import org.onosproject.net.behaviour.PipelinerContext;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.TunnelIdCriterion;
+import org.onosproject.net.flow.criteria.VlanIdCriterion;
+import org.onosproject.net.flow.instructions.Instruction;
+import org.onosproject.net.flow.instructions.Instructions;
+import org.onosproject.net.flow.instructions.L2ModificationInstruction;
+import org.onosproject.net.flowobjective.DefaultNextObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveStore;
+import org.onosproject.net.flowobjective.NextObjective;
+import org.onosproject.net.flowobjective.ObjectiveContext;
+import org.onosproject.net.flowobjective.ObjectiveError;
+import org.onosproject.net.group.DefaultGroupBucket;
+import org.onosproject.net.group.DefaultGroupDescription;
+import org.onosproject.net.group.DefaultGroupKey;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupBucket;
+import org.onosproject.net.group.GroupBuckets;
+import org.onosproject.net.group.GroupDescription;
+import org.onosproject.net.group.GroupEvent;
+import org.onosproject.net.group.GroupKey;
+import org.onosproject.net.group.GroupListener;
+import org.onosproject.net.group.GroupService;
+import org.onosproject.store.service.AtomicCounter;
+import org.onosproject.store.service.StorageService;
+import org.slf4j.Logger;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.driver.pipeline.ofdpa.Ofdpa2Pipeline.*;
+import static org.onosproject.driver.pipeline.ofdpa.OfdpaGroupHandlerUtility.*;
+import static org.onosproject.net.flow.criteria.Criterion.Type.TUNNEL_ID;
+import static org.onosproject.net.flow.criteria.Criterion.Type.VLAN_VID;
+import static org.onosproject.net.group.GroupDescription.Type.ALL;
+import static org.onosproject.net.group.GroupDescription.Type.SELECT;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Group handler that emulates Broadcom OF-DPA TTP.
+ */
+public class Ofdpa2GroupHandler {
+ protected final Logger log = getLogger(getClass());
+
+ // Services, Stores
+ protected GroupService groupService;
+ protected StorageService storageService;
+ protected FlowObjectiveStore flowObjectiveStore;
+
+ // index number for group creation
+ private AtomicCounter nextIndex;
+
+ protected DeviceId deviceId;
+ private Cache<GroupKey, List<OfdpaGroupHandlerUtility.OfdpaNextGroup>> pendingAddNextObjectives;
+ private Cache<NextObjective, List<GroupKey>> pendingRemoveNextObjectives;
+ private Cache<GroupKey, Set<OfdpaGroupHandlerUtility.GroupChainElem>> pendingGroups;
+ private ConcurrentHashMap<GroupKey, Set<NextObjective>> pendingUpdateNextObjectives;
+
+ // local store for pending bucketAdds - by design there can be multiple
+ // pending bucket for a group
+ protected ConcurrentHashMap<Integer, Set<NextObjective>> pendingBuckets =
+ new ConcurrentHashMap<>();
+
+ private ScheduledExecutorService groupCheckerExecutor =
+ Executors.newScheduledThreadPool(2, groupedThreads("onos/pipeliner", "ofdpa-%d", log));
+
+ public Cache<GroupKey, List<OfdpaNextGroup>> pendingAddNextObjectives() {
+ return pendingAddNextObjectives;
+ }
+
+ public Cache<GroupKey, Set<GroupChainElem>> pendingGroups() {
+ return pendingGroups;
+ }
+
+ /**
+ * Determines whether this pipeline support copy ttl instructions or not.
+ *
+ * @return true if copy ttl instructions are supported
+ */
+ protected boolean supportCopyTtl() {
+ return true;
+ }
+
+ /**
+ * Determines whether this pipeline support set mpls bos instruction or not.
+ *
+ * @return true if set mpls bos instruction is supported
+ */
+ protected boolean supportSetMplsBos() {
+ return true;
+ }
+
+ /**
+ * Determines whether this pipeline requires popping VLAN before pushing MPLS.
+ * <p>
+ * If required, pop vlan before push mpls and add an arbitrary vlan back afterward.
+ * MPLS interface group will substitute the arbitrary VLAN with expected VLAN later on.
+ *
+ * @return true if this pipeline requires popping VLAN before pushing MPLS
+ */
+ protected boolean requireVlanPopBeforeMplsPush() {
+ return false;
+ }
+
+ protected void init(DeviceId deviceId, PipelinerContext context) {
+ ServiceDirectory serviceDirectory = context.directory();
+ this.deviceId = deviceId;
+ this.flowObjectiveStore = context.store();
+ this.groupService = serviceDirectory.get(GroupService.class);
+ this.storageService = serviceDirectory.get(StorageService.class);
+ this.nextIndex = storageService.getAtomicCounter("group-id-index-counter");
+
+ pendingAddNextObjectives = CacheBuilder.newBuilder()
+ .expireAfterWrite(20, TimeUnit.SECONDS)
+ .removalListener((RemovalNotification<GroupKey, List<OfdpaNextGroup>> notification) -> {
+ if (notification.getCause() == RemovalCause.EXPIRED &&
+ Objects.nonNull(notification.getValue())) {
+ notification.getValue()
+ .forEach(ofdpaNextGrp ->
+ fail(ofdpaNextGrp.nextObjective(),
+ ObjectiveError.GROUPINSTALLATIONFAILED));
+ }
+ }).build();
+
+ pendingRemoveNextObjectives = CacheBuilder.newBuilder()
+ .expireAfterWrite(20, TimeUnit.SECONDS)
+ .removalListener((RemovalNotification<NextObjective, List<GroupKey>> notification) -> {
+ if (notification.getCause() == RemovalCause.EXPIRED) {
+ fail(notification.getKey(),
+ ObjectiveError.GROUPREMOVALFAILED);
+ }
+ }).build();
+ pendingGroups = CacheBuilder.newBuilder()
+ .expireAfterWrite(20, TimeUnit.SECONDS)
+ .removalListener((RemovalNotification<GroupKey, Set<GroupChainElem>> notification) -> {
+ if (notification.getCause() == RemovalCause.EXPIRED) {
+ log.error("Unable to install group with key {} and pending GCEs: {}",
+ notification.getKey(), notification.getValue());
+ }
+ }).build();
+ pendingUpdateNextObjectives = new ConcurrentHashMap<>();
+ GroupChecker groupChecker = new GroupChecker(this);
+ groupCheckerExecutor.scheduleAtFixedRate(groupChecker, 0, 500, TimeUnit.MILLISECONDS);
+ groupService.addListener(new InnerGroupListener());
+ }
+
+ //////////////////////////////////////
+ // Group Creation
+ //////////////////////////////////////
+
+ /**
+ * Adds a list of group chain by given NextObjective.
+ *
+ * @param nextObjective the NextObjective
+ */
+ protected void addGroup(NextObjective nextObjective) {
+ switch (nextObjective.type()) {
+ case SIMPLE:
+ Collection<TrafficTreatment> treatments = nextObjective.next();
+ if (treatments.size() != 1) {
+ log.error("Next Objectives of type Simple should only have a "
+ + "single Traffic Treatment. Next Objective Id:{}",
+ nextObjective.id());
+ fail(nextObjective, ObjectiveError.BADPARAMS);
+ return;
+ }
+ processSimpleNextObjective(nextObjective);
+ break;
+ case BROADCAST:
+ processBroadcastNextObjective(nextObjective);
+ break;
+ case HASHED:
+ if (!verifyHashedNextObjective(nextObjective)) {
+ log.error("Next Objectives of type hashed not supported. Next Objective Id:{}",
+ nextObjective.id());
+ fail(nextObjective, ObjectiveError.BADPARAMS);
+ return;
+ }
+ processHashedNextObjective(nextObjective);
+ break;
+ case FAILOVER:
+ fail(nextObjective, ObjectiveError.UNSUPPORTED);
+ log.warn("Unsupported next objective type {}", nextObjective.type());
+ break;
+ default:
+ fail(nextObjective, ObjectiveError.UNKNOWN);
+ log.warn("Unknown next objective type {}", nextObjective.type());
+ }
+ }
+
+ /**
+ * As per the OFDPA 2.0 TTP, packets are sent out of ports by using
+ * a chain of groups. The simple Next Objective passed
+ * in by the application has to be broken up into a group chain
+ * comprising of an L3 Unicast Group that points to an L2 Interface
+ * Group which in-turn points to an output port. In some cases, the simple
+ * next Objective can just be an L2 interface without the need for chaining.
+ *
+ * @param nextObj the nextObjective of type SIMPLE
+ */
+ private void processSimpleNextObjective(NextObjective nextObj) {
+ TrafficTreatment treatment = nextObj.next().iterator().next();
+ // determine if plain L2 or L3->L2
+ boolean plainL2 = true;
+ for (Instruction ins : treatment.allInstructions()) {
+ if (ins.type() == Instruction.Type.L2MODIFICATION) {
+ L2ModificationInstruction l2ins = (L2ModificationInstruction) ins;
+ if (l2ins.subtype() == L2ModificationInstruction.L2SubType.ETH_DST ||
+ l2ins.subtype() == L2ModificationInstruction.L2SubType.ETH_SRC) {
+ plainL2 = false;
+ break;
+ }
+ }
+ }
+
+ if (plainL2) {
+ createL2InterfaceGroup(nextObj);
+ return;
+ }
+
+ boolean isMpls = false;
+ // In order to understand if it is a pseudo wire related
+ // next objective we look for the tunnel id in the meta.
+ boolean isPw = false;
+ if (nextObj.meta() != null) {
+ isMpls = isNotMplsBos(nextObj.meta());
+
+ TunnelIdCriterion tunnelIdCriterion = (TunnelIdCriterion) nextObj
+ .meta()
+ .getCriterion(TUNNEL_ID);
+ if (tunnelIdCriterion != null) {
+ isPw = true;
+ }
+
+ }
+
+ if (!isPw) {
+ // break up simple next objective to GroupChain objects
+ GroupInfo groupInfo = createL2L3Chain(treatment, nextObj.id(),
+ nextObj.appId(), isMpls,
+ nextObj.meta());
+ if (groupInfo == null) {
+ log.error("Could not process nextObj={} in dev:{}", nextObj.id(), deviceId);
+ return;
+ }
+ // create object for local and distributed storage
+ Deque<GroupKey> gkeyChain = new ArrayDeque<>();
+ gkeyChain.addFirst(groupInfo.innerMostGroupDesc().appCookie());
+ gkeyChain.addFirst(groupInfo.nextGroupDesc().appCookie());
+ OfdpaNextGroup ofdpaGrp =
+ new OfdpaNextGroup(Collections.singletonList(gkeyChain), nextObj);
+
+ // store l3groupkey with the ofdpaNextGroup for the nextObjective that depends on it
+ updatePendingNextObjective(groupInfo.nextGroupDesc().appCookie(), ofdpaGrp);
+
+ // now we are ready to send the l2 groupDescription (inner), as all the stores
+ // that will get async replies have been updated. By waiting to update
+ // the stores, we prevent nasty race conditions.
+ groupService.addGroup(groupInfo.innerMostGroupDesc());
+ } else {
+ // We handle the pseudo wire with a different a procedure.
+ // This procedure is meant to handle both initiation and
+ // termination of the pseudo wire.
+ processPwNextObjective(nextObj);
+ }
+ }
+
+ /**
+ * Creates a simple L2 Interface Group.
+ *
+ * @param nextObj the next Objective
+ */
+ private void createL2InterfaceGroup(NextObjective nextObj) {
+ VlanId assignedVlan = readVlanFromSelector(nextObj.meta());
+ if (assignedVlan == null) {
+ log.warn("VLAN ID required by simple next obj is missing. Abort.");
+ fail(nextObj, ObjectiveError.BADPARAMS);
+ return;
+ }
+
+ List<GroupInfo> groupInfos = prepareL2InterfaceGroup(nextObj, assignedVlan);
+
+ // There is only one L2 interface group in this case
+ GroupDescription l2InterfaceGroupDesc = groupInfos.get(0).innerMostGroupDesc();
+
+ // Put all dependency information into allGroupKeys
+ List<Deque<GroupKey>> allGroupKeys = Lists.newArrayList();
+ Deque<GroupKey> gkeyChain = new ArrayDeque<>();
+ gkeyChain.addFirst(l2InterfaceGroupDesc.appCookie());
+ allGroupKeys.add(gkeyChain);
+
+ // Point the next objective to this group
+ OfdpaNextGroup ofdpaGrp = new OfdpaNextGroup(allGroupKeys, nextObj);
+ updatePendingNextObjective(l2InterfaceGroupDesc.appCookie(), ofdpaGrp);
+
+ // Start installing the inner-most group
+ groupService.addGroup(l2InterfaceGroupDesc);
+ }
+
+ /**
+ * Creates one of two possible group-chains from the treatment
+ * passed in. Depending on the MPLS boolean, this method either creates
+ * an L3Unicast Group --> L2Interface Group, if mpls is false;
+ * or MPLSInterface Group --> L2Interface Group, if mpls is true;
+ * The returned 'inner' group description is always the L2 Interface group.
+ *
+ * @param treatment that needs to be broken up to create the group chain
+ * @param nextId of the next objective that needs this group chain
+ * @param appId of the application that sent this next objective
+ * @param mpls determines if L3Unicast or MPLSInterface group is created
+ * @param meta metadata passed in by the application as part of the nextObjective
+ * @return GroupInfo containing the GroupDescription of the
+ * L2Interface group(inner) and the GroupDescription of the (outer)
+ * L3Unicast/MPLSInterface group. May return null if there is an
+ * error in processing the chain
+ */
+ protected GroupInfo createL2L3Chain(TrafficTreatment treatment, int nextId,
+ ApplicationId appId, boolean mpls,
+ TrafficSelector meta) {
+ return createL2L3ChainInternal(treatment, nextId, appId, mpls, meta, true);
+ }
+
+ /**
+ * Internal implementation of createL2L3Chain.
+ * <p>
+ * The is_present bit in set_vlan_vid action is required to be 0 in OFDPA i12.
+ * Since it is non-OF spec, we need an extension treatment for that.
+ * The useSetVlanExtension must be set to false for OFDPA i12.
+ * </p>
+ *
+ * @param treatment that needs to be broken up to create the group chain
+ * @param nextId of the next objective that needs this group chain
+ * @param appId of the application that sent this next objective
+ * @param mpls determines if L3Unicast or MPLSInterface group is created
+ * @param meta metadata passed in by the application as part of the nextObjective
+ * @param useSetVlanExtension use the setVlanVid extension that has is_present bit set to 0.
+ * @return GroupInfo containing the GroupDescription of the
+ * L2Interface group(inner) and the GroupDescription of the (outer)
+ * L3Unicast/MPLSInterface group. May return null if there is an
+ * error in processing the chain
+ */
+ protected GroupInfo createL2L3ChainInternal(TrafficTreatment treatment, int nextId,
+ ApplicationId appId, boolean mpls,
+ TrafficSelector meta, boolean useSetVlanExtension) {
+ // for the l2interface group, get vlan and port info
+ // for the outer group, get the src/dst mac, and vlan info
+ TrafficTreatment.Builder outerTtb = DefaultTrafficTreatment.builder();
+ TrafficTreatment.Builder innerTtb = DefaultTrafficTreatment.builder();
+ VlanId vlanid = null;
+ long portNum = 0;
+ boolean setVlan = false, popVlan = false;
+ MacAddress srcMac;
+ MacAddress dstMac;
+ for (Instruction ins : treatment.allInstructions()) {
+ if (ins.type() == Instruction.Type.L2MODIFICATION) {
+ L2ModificationInstruction l2ins = (L2ModificationInstruction) ins;
+ switch (l2ins.subtype()) {
+ case ETH_DST:
+ dstMac = ((L2ModificationInstruction.ModEtherInstruction) l2ins).mac();
+ outerTtb.setEthDst(dstMac);
+ break;
+ case ETH_SRC:
+ srcMac = ((L2ModificationInstruction.ModEtherInstruction) l2ins).mac();
+ outerTtb.setEthSrc(srcMac);
+ break;
+ case VLAN_ID:
+ vlanid = ((L2ModificationInstruction.ModVlanIdInstruction) l2ins).vlanId();
+ if (useSetVlanExtension) {
+ OfdpaSetVlanVid ofdpaSetVlanVid = new OfdpaSetVlanVid(vlanid);
+ outerTtb.extension(ofdpaSetVlanVid, deviceId);
+ } else {
+ outerTtb.setVlanId(vlanid);
+ }
+ setVlan = true;
+ break;
+ case VLAN_POP:
+ innerTtb.popVlan();
+ popVlan = true;
+ break;
+ case DEC_MPLS_TTL:
+ case MPLS_LABEL:
+ case MPLS_POP:
+ case MPLS_PUSH:
+ case VLAN_PCP:
+ case VLAN_PUSH:
+ default:
+ break;
+ }
+ } else if (ins.type() == Instruction.Type.OUTPUT) {
+ portNum = ((Instructions.OutputInstruction) ins).port().toLong();
+ innerTtb.add(ins);
+ } else {
+ log.warn("Driver does not handle this type of TrafficTreatment"
+ + " instruction in nextObjectives: {}", ins.type());
+ }
+ }
+
+ if (vlanid == null && meta != null) {
+ // use metadata if available
+ Criterion vidCriterion = meta.getCriterion(VLAN_VID);
+ if (vidCriterion != null) {
+ vlanid = ((VlanIdCriterion) vidCriterion).vlanId();
+ }
+ // if vlan is not set, use the vlan in metadata for outerTtb
+ if (vlanid != null && !setVlan) {
+ if (useSetVlanExtension) {
+ OfdpaSetVlanVid ofdpaSetVlanVid = new OfdpaSetVlanVid(vlanid);
+ outerTtb.extension(ofdpaSetVlanVid, deviceId);
+ } else {
+ outerTtb.setVlanId(vlanid);
+ }
+ }
+ }
+
+ if (vlanid == null) {
+ log.error("Driver cannot process an L2/L3 group chain without "
+ + "egress vlan information for dev: {} port:{}",
+ deviceId, portNum);
+ return null;
+ }
+
+ if (!setVlan && !popVlan) {
+ // untagged outgoing port
+ TrafficTreatment.Builder temp = DefaultTrafficTreatment.builder();
+ temp.popVlan();
+ innerTtb.build().allInstructions().forEach(temp::add);
+ innerTtb = temp;
+ }
+
+ // assemble information for ofdpa l2interface group
+ int l2groupId = l2GroupId(vlanid, portNum);
+ // a globally unique groupkey that is different for ports in the same device,
+ // but different for the same portnumber on different devices. Also different
+ // for the various group-types created out of the same next objective.
+ int l2gk = l2InterfaceGroupKey(deviceId, vlanid, portNum);
+ final GroupKey l2groupkey = new DefaultGroupKey(appKryo.serialize(l2gk));
+
+ // assemble information for outer group
+ GroupDescription outerGrpDesc;
+ if (mpls) {
+ // outer group is MPLS Interface
+ int mplsInterfaceIndex = getNextAvailableIndex();
+ int mplsGroupId = MPLS_INTERFACE_TYPE | (SUBTYPE_MASK & mplsInterfaceIndex);
+ final GroupKey mplsGroupKey = new DefaultGroupKey(
+ appKryo.serialize(mplsInterfaceIndex));
+ outerTtb.group(new GroupId(l2groupId));
+ // create the mpls-interface group description to wait for the
+ // l2 interface group to be processed
+ GroupBucket mplsinterfaceGroupBucket =
+ DefaultGroupBucket.createIndirectGroupBucket(outerTtb.build());
+ outerGrpDesc = new DefaultGroupDescription(
+ deviceId,
+ GroupDescription.Type.INDIRECT,
+ new GroupBuckets(Collections.singletonList(
+ mplsinterfaceGroupBucket)),
+ mplsGroupKey,
+ mplsGroupId,
+ appId);
+ log.debug("Trying MPLS-Interface: device:{} gid:{} gkey:{} nextid:{}",
+ deviceId, Integer.toHexString(mplsGroupId),
+ mplsGroupKey, nextId);
+ } else {
+ // outer group is L3Unicast
+ int l3unicastIndex = getNextAvailableIndex();
+ int l3groupId = L3_UNICAST_TYPE | (TYPE_MASK & l3unicastIndex);
+ final GroupKey l3groupkey = new DefaultGroupKey(
+ appKryo.serialize(l3unicastIndex));
+ outerTtb.group(new GroupId(l2groupId));
+ // create the l3unicast group description to wait for the
+ // l2 interface group to be processed
+ GroupBucket l3unicastGroupBucket =
+ DefaultGroupBucket.createIndirectGroupBucket(outerTtb.build());
+ outerGrpDesc = new DefaultGroupDescription(
+ deviceId,
+ GroupDescription.Type.INDIRECT,
+ new GroupBuckets(Collections.singletonList(l3unicastGroupBucket)),
+ l3groupkey,
+ l3groupId,
+ appId);
+ log.debug("Trying L3Unicast: device:{} gid:{} gkey:{} nextid:{}",
+ deviceId, Integer.toHexString(l3groupId),
+ l3groupkey, nextId);
+ }
+
+ // store l2groupkey with the groupChainElem for the outer-group that depends on it
+ GroupChainElem gce = new GroupChainElem(outerGrpDesc, 1, false, deviceId);
+ updatePendingGroups(l2groupkey, gce);
+
+ // create group description for the inner l2 interface group
+ GroupBucket l2InterfaceGroupBucket =
+ DefaultGroupBucket.createIndirectGroupBucket(innerTtb.build());
+ GroupDescription l2groupDescription =
+ new DefaultGroupDescription(deviceId,
+ GroupDescription.Type.INDIRECT,
+ new GroupBuckets(Collections.singletonList(l2InterfaceGroupBucket)),
+ l2groupkey,
+ l2groupId,
+ appId);
+ log.debug("Trying L2Interface: device:{} gid:{} gkey:{} nextId:{}",
+ deviceId, Integer.toHexString(l2groupId),
+ l2groupkey, nextId);
+ return new GroupInfo(l2groupDescription, outerGrpDesc);
+
+ }
+
+ /**
+ * As per the OFDPA 2.0 TTP, packets are sent out of ports by using
+ * a chain of groups. The broadcast Next Objective passed in by the application
+ * has to be broken up into a group chain comprising of an
+ * L2 Flood group or L3 Multicast group, whose buckets point to L2 Interface groups.
+ *
+ * @param nextObj the nextObjective of type BROADCAST
+ */
+ private void processBroadcastNextObjective(NextObjective nextObj) {
+ VlanId assignedVlan = readVlanFromSelector(nextObj.meta());
+ if (assignedVlan == null) {
+ log.warn("VLAN ID required by broadcast next obj is missing. Abort.");
+ fail(nextObj, ObjectiveError.BADPARAMS);
+ return;
+ }
+
+ List<GroupInfo> groupInfos = prepareL2InterfaceGroup(nextObj, assignedVlan);
+
+ IpPrefix ipDst = readIpDstFromSelector(nextObj.meta());
+ if (ipDst != null) {
+ if (ipDst.isMulticast()) {
+ createL3MulticastGroup(nextObj, assignedVlan, groupInfos);
+ } else {
+ log.warn("Broadcast NextObj with non-multicast IP address {}", nextObj);
+ fail(nextObj, ObjectiveError.BADPARAMS);
+ }
+ } else {
+ createL2FloodGroup(nextObj, assignedVlan, groupInfos);
+ }
+ }
+
+ private List<GroupInfo> prepareL2InterfaceGroup(NextObjective nextObj,
+ VlanId assignedVlan) {
+ ImmutableList.Builder<GroupInfo> groupInfoBuilder = ImmutableList.builder();
+
+ // break up broadcast next objective to multiple groups
+ Collection<TrafficTreatment> buckets = nextObj.next();
+
+ // each treatment is converted to an L2 interface group
+ for (TrafficTreatment treatment : buckets) {
+ TrafficTreatment.Builder newTreatment = DefaultTrafficTreatment.builder();
+ PortNumber portNum = null;
+ VlanId egressVlan = null;
+ // ensure that the only allowed treatments are pop-vlan and output
+ for (Instruction ins : treatment.allInstructions()) {
+ if (ins.type() == Instruction.Type.L2MODIFICATION) {
+ L2ModificationInstruction l2ins = (L2ModificationInstruction) ins;
+ switch (l2ins.subtype()) {
+ case VLAN_POP:
+ newTreatment.add(l2ins);
+ break;
+ case VLAN_ID:
+ egressVlan = ((L2ModificationInstruction.ModVlanIdInstruction) l2ins).vlanId();
+ break;
+ default:
+ log.debug("action {} not permitted for broadcast nextObj",
+ l2ins.subtype());
+ break;
+ }
+ } else if (ins.type() == Instruction.Type.OUTPUT) {
+ portNum = ((Instructions.OutputInstruction) ins).port();
+ newTreatment.add(ins);
+ } else {
+ log.debug("TrafficTreatment of type {} not permitted in " +
+ " broadcast nextObjective", ins.type());
+ }
+ }
+
+ if (portNum == null) {
+ log.warn("Can't find output port for the bucket {}.", treatment);
+ continue;
+ }
+
+ // assemble info for l2 interface group
+ VlanId l2InterfaceGroupVlan =
+ (egressVlan != null && !assignedVlan.equals(egressVlan)) ?
+ egressVlan : assignedVlan;
+ int l2gk = l2InterfaceGroupKey(deviceId, l2InterfaceGroupVlan, portNum.toLong());
+ final GroupKey l2InterfaceGroupKey =
+ new DefaultGroupKey(appKryo.serialize(l2gk));
+ int l2InterfaceGroupId = L2_INTERFACE_TYPE |
+ ((l2InterfaceGroupVlan.toShort() & THREE_BIT_MASK) << PORT_LEN) |
+ ((int) portNum.toLong() & FOUR_BIT_MASK);
+ GroupBucket l2InterfaceGroupBucket =
+ DefaultGroupBucket.createIndirectGroupBucket(newTreatment.build());
+ GroupDescription l2InterfaceGroupDescription =
+ new DefaultGroupDescription(deviceId,
+ GroupDescription.Type.INDIRECT,
+ new GroupBuckets(Collections.singletonList(
+ l2InterfaceGroupBucket)),
+ l2InterfaceGroupKey,
+ l2InterfaceGroupId,
+ nextObj.appId());
+ log.debug("Trying L2-Interface: device:{} gid:{} gkey:{} nextid:{}",
+ deviceId, Integer.toHexString(l2InterfaceGroupId),
+ l2InterfaceGroupKey, nextObj.id());
+
+ groupInfoBuilder.add(new GroupInfo(l2InterfaceGroupDescription,
+ l2InterfaceGroupDescription));
+ }
+ return groupInfoBuilder.build();
+ }
+
+ private void createL2FloodGroup(NextObjective nextObj, VlanId vlanId,
+ List<GroupInfo> groupInfos) {
+ // assemble info for l2 flood group. Since there can be only one flood
+ // group for a vlan, its index is always the same - 0
+ Integer l2FloodGroupId = L2_FLOOD_TYPE | (vlanId.toShort() << 16);
+ final GroupKey l2FloodGroupKey = l2FloodGroupKey(vlanId, deviceId);
+
+ // collection of group buckets pointing to all the l2 interface groups
+ List<GroupBucket> l2floodBuckets = generateNextGroupBuckets(groupInfos, ALL);
+ // create the l2flood group-description to wait for all the
+ // l2interface groups to be processed
+ GroupDescription l2floodGroupDescription =
+ new DefaultGroupDescription(
+ deviceId,
+ ALL,
+ new GroupBuckets(l2floodBuckets),
+ l2FloodGroupKey,
+ l2FloodGroupId,
+ nextObj.appId());
+ log.debug("Trying L2-Flood: device:{} gid:{} gkey:{} nextid:{}",
+ deviceId, Integer.toHexString(l2FloodGroupId),
+ l2FloodGroupKey, nextObj.id());
+
+ // Put all dependency information into allGroupKeys
+ List<Deque<GroupKey>> allGroupKeys = Lists.newArrayList();
+ groupInfos.forEach(groupInfo -> {
+ Deque<GroupKey> groupKeyChain = new ArrayDeque<>();
+ // In this case we should have L2 interface group only
+ groupKeyChain.addFirst(groupInfo.nextGroupDesc().appCookie());
+ groupKeyChain.addFirst(l2FloodGroupKey);
+ allGroupKeys.add(groupKeyChain);
+ });
+
+ // Point the next objective to this group
+ OfdpaNextGroup ofdpaGrp = new OfdpaNextGroup(allGroupKeys, nextObj);
+ updatePendingNextObjective(l2FloodGroupKey, ofdpaGrp);
+
+ GroupChainElem gce = new GroupChainElem(l2floodGroupDescription,
+ groupInfos.size(), false, deviceId);
+ groupInfos.forEach(groupInfo -> {
+ // Point this group to the next group
+ updatePendingGroups(groupInfo.nextGroupDesc().appCookie(), gce);
+ // Start installing the inner-most group
+ groupService.addGroup(groupInfo.innerMostGroupDesc());
+ });
+ }
+
+ private void createL3MulticastGroup(NextObjective nextObj, VlanId vlanId,
+ List<GroupInfo> groupInfos) {
+ List<GroupBucket> l3McastBuckets = new ArrayList<>();
+ groupInfos.forEach(groupInfo -> {
+ // Points to L3 interface group if there is one.
+ // Otherwise points to L2 interface group directly.
+ GroupDescription nextGroupDesc = (groupInfo.nextGroupDesc() != null) ?
+ groupInfo.nextGroupDesc() : groupInfo.innerMostGroupDesc();
+ TrafficTreatment.Builder ttb = DefaultTrafficTreatment.builder();
+ ttb.group(new GroupId(nextGroupDesc.givenGroupId()));
+ GroupBucket abucket = DefaultGroupBucket.createAllGroupBucket(ttb.build());
+ l3McastBuckets.add(abucket);
+ });
+
+ int l3MulticastIndex = getNextAvailableIndex();
+ int l3MulticastGroupId = L3_MULTICAST_TYPE |
+ vlanId.toShort() << 16 | (TYPE_VLAN_MASK & l3MulticastIndex);
+ final GroupKey l3MulticastGroupKey =
+ new DefaultGroupKey(appKryo.serialize(l3MulticastIndex));
+
+ GroupDescription l3MulticastGroupDesc = new DefaultGroupDescription(deviceId,
+ ALL,
+ new GroupBuckets(l3McastBuckets),
+ l3MulticastGroupKey,
+ l3MulticastGroupId,
+ nextObj.appId());
+
+ // Put all dependency information into allGroupKeys
+ List<Deque<GroupKey>> allGroupKeys = Lists.newArrayList();
+ groupInfos.forEach(groupInfo -> {
+ Deque<GroupKey> gkeyChain = new ArrayDeque<>();
+ gkeyChain.addFirst(groupInfo.innerMostGroupDesc().appCookie());
+ // Add L3 interface group to the chain if there is one.
+ if (!groupInfo.nextGroupDesc().equals(groupInfo.innerMostGroupDesc())) {
+ gkeyChain.addFirst(groupInfo.nextGroupDesc().appCookie());
+ }
+ gkeyChain.addFirst(l3MulticastGroupKey);
+ allGroupKeys.add(gkeyChain);
+ });
+
+ // Point the next objective to this group
+ OfdpaNextGroup ofdpaGrp = new OfdpaNextGroup(allGroupKeys, nextObj);
+ updatePendingNextObjective(l3MulticastGroupKey, ofdpaGrp);
+
+ GroupChainElem outerGce = new GroupChainElem(l3MulticastGroupDesc,
+ groupInfos.size(), false, deviceId);
+ groupInfos.forEach(groupInfo -> {
+ // Point this group (L3 multicast) to the next group
+ updatePendingGroups(groupInfo.nextGroupDesc().appCookie(), outerGce);
+
+ // Point next group to inner-most group, if any
+ if (!groupInfo.nextGroupDesc().equals(groupInfo.innerMostGroupDesc())) {
+ GroupChainElem innerGce = new GroupChainElem(groupInfo.nextGroupDesc(),
+ 1, false, deviceId);
+ updatePendingGroups(groupInfo.innerMostGroupDesc().appCookie(), innerGce);
+ }
+
+ // Start installing the inner-most group
+ groupService.addGroup(groupInfo.innerMostGroupDesc());
+ });
+ }
+
+ /**
+ * As per the OFDPA 2.0 TTP, packets are sent out of ports by using
+ * a chain of groups. The hashed Next Objective passed in by the application
+ * has to be broken up into a group chain comprising of an
+ * L3 ECMP group as the top level group. Buckets of this group can point
+ * to a variety of groups in a group chain, depending on the whether
+ * MPLS labels are being pushed or not.
+ * <p>
+ * NOTE: We do not create MPLS ECMP groups as they are unimplemented in
+ * OF-DPA 2.0 (even though it is in the spec). Therefore we do not
+ * check the nextObjective meta to see what is matching before being
+ * sent to this nextObjective.
+ *
+ * @param nextObj the nextObjective of type HASHED
+ */
+ protected void processHashedNextObjective(NextObjective nextObj) {
+ // storage for all group keys in the chain of groups created
+ List<Deque<GroupKey>> allGroupKeys = new ArrayList<>();
+ List<GroupInfo> unsentGroups = new ArrayList<>();
+ createHashBucketChains(nextObj, allGroupKeys, unsentGroups);
+
+ // now we can create the outermost L3 ECMP group
+ List<GroupBucket> l3ecmpGroupBuckets = new ArrayList<>();
+ for (GroupInfo gi : unsentGroups) {
+ // create ECMP bucket to point to the outer group
+ TrafficTreatment.Builder ttb = DefaultTrafficTreatment.builder();
+ ttb.group(new GroupId(gi.nextGroupDesc().givenGroupId()));
+ GroupBucket sbucket = DefaultGroupBucket
+ .createSelectGroupBucket(ttb.build());
+ l3ecmpGroupBuckets.add(sbucket);
+ }
+ int l3ecmpIndex = getNextAvailableIndex();
+ int l3ecmpGroupId = L3_ECMP_TYPE | (TYPE_MASK & l3ecmpIndex);
+ GroupKey l3ecmpGroupKey = new DefaultGroupKey(
+ appKryo.serialize(l3ecmpIndex));
+ GroupDescription l3ecmpGroupDesc =
+ new DefaultGroupDescription(
+ deviceId,
+ SELECT,
+ new GroupBuckets(l3ecmpGroupBuckets),
+ l3ecmpGroupKey,
+ l3ecmpGroupId,
+ nextObj.appId());
+ GroupChainElem l3ecmpGce = new GroupChainElem(l3ecmpGroupDesc,
+ l3ecmpGroupBuckets.size(),
+ false, deviceId);
+
+ // create objects for local and distributed storage
+ allGroupKeys.forEach(gKeyChain -> gKeyChain.addFirst(l3ecmpGroupKey));
+ OfdpaNextGroup ofdpaGrp = new OfdpaNextGroup(allGroupKeys, nextObj);
+
+ // store l3ecmpGroupKey with the ofdpaGroupChain for the nextObjective
+ // that depends on it
+ updatePendingNextObjective(l3ecmpGroupKey, ofdpaGrp);
+
+ log.debug("Trying L3ECMP: device:{} gid:{} gkey:{} nextId:{}",
+ deviceId, Integer.toHexString(l3ecmpGroupId),
+ l3ecmpGroupKey, nextObj.id());
+ // finally we are ready to send the innermost groups
+ for (GroupInfo gi : unsentGroups) {
+ log.debug("Sending innermost group {} in group chain on device {} ",
+ Integer.toHexString(gi.innerMostGroupDesc().givenGroupId()), deviceId);
+ updatePendingGroups(gi.nextGroupDesc().appCookie(), l3ecmpGce);
+ groupService.addGroup(gi.innerMostGroupDesc());
+ }
+ }
+
+ /**
+ * Creates group chains for all buckets in a hashed group, and stores the
+ * GroupInfos and GroupKeys for all the groups in the lists passed in, which
+ * should be empty.
+ * <p>
+ * Does not create the top level ECMP group. Does not actually send the
+ * groups to the groupService.
+ *
+ * @param nextObj the Next Objective with buckets that need to be converted
+ * to group chains
+ * @param allGroupKeys a list to store groupKey for each bucket-group-chain
+ * @param unsentGroups a list to store GroupInfo for each bucket-group-chain
+ */
+ protected void createHashBucketChains(NextObjective nextObj,
+ List<Deque<GroupKey>> allGroupKeys,
+ List<GroupInfo> unsentGroups) {
+ // break up hashed next objective to multiple groups
+ Collection<TrafficTreatment> buckets = nextObj.next();
+
+ for (TrafficTreatment bucket : buckets) {
+ //figure out how many labels are pushed in each bucket
+ int labelsPushed = 0;
+ MplsLabel innermostLabel = null;
+ for (Instruction ins : bucket.allInstructions()) {
+ if (ins.type() == Instruction.Type.L2MODIFICATION) {
+ L2ModificationInstruction l2ins = (L2ModificationInstruction) ins;
+ if (l2ins.subtype() == L2ModificationInstruction.L2SubType.MPLS_PUSH) {
+ labelsPushed++;
+ }
+ if (l2ins.subtype() == L2ModificationInstruction.L2SubType.MPLS_LABEL) {
+ if (innermostLabel == null) {
+ innermostLabel =
+ ((L2ModificationInstruction.ModMplsLabelInstruction) l2ins).label();
+ }
+ }
+ }
+ }
+
+ Deque<GroupKey> gKeyChain = new ArrayDeque<>();
+ // XXX we only deal with 0 and 1 label push right now
+ if (labelsPushed == 0) {
+ GroupInfo noLabelGroupInfo;
+ TrafficSelector metaSelector = nextObj.meta();
+ if (metaSelector != null) {
+ if (isNotMplsBos(metaSelector)) {
+ noLabelGroupInfo = createL2L3Chain(bucket, nextObj.id(),
+ nextObj.appId(), true,
+ nextObj.meta());
+ } else {
+ noLabelGroupInfo = createL2L3Chain(bucket, nextObj.id(),
+ nextObj.appId(), false,
+ nextObj.meta());
+ }
+ } else {
+ noLabelGroupInfo = createL2L3Chain(bucket, nextObj.id(),
+ nextObj.appId(), false,
+ nextObj.meta());
+ }
+ if (noLabelGroupInfo == null) {
+ log.error("Could not process nextObj={} in dev:{}",
+ nextObj.id(), deviceId);
+ return;
+ }
+ gKeyChain.addFirst(noLabelGroupInfo.innerMostGroupDesc().appCookie());
+ gKeyChain.addFirst(noLabelGroupInfo.nextGroupDesc().appCookie());
+
+ // we can't send the inner group description yet, as we have to
+ // create the dependent ECMP group first. So we store..
+ unsentGroups.add(noLabelGroupInfo);
+
+ } else if (labelsPushed == 1) {
+ GroupInfo onelabelGroupInfo = createL2L3Chain(bucket, nextObj.id(),
+ nextObj.appId(), true,
+ nextObj.meta());
+ if (onelabelGroupInfo == null) {
+ log.error("Could not process nextObj={} in dev:{}",
+ nextObj.id(), deviceId);
+ return;
+ }
+ // we need to add another group to this chain - the L3VPN group
+ TrafficTreatment.Builder l3vpnTtb = DefaultTrafficTreatment.builder();
+ if (requireVlanPopBeforeMplsPush()) {
+ l3vpnTtb.popVlan();
+ }
+ l3vpnTtb.pushMpls()
+ .setMpls(innermostLabel)
+ .group(new GroupId(onelabelGroupInfo.nextGroupDesc().givenGroupId()));
+ if (supportCopyTtl()) {
+ l3vpnTtb.copyTtlOut();
+ }
+ if (supportSetMplsBos()) {
+ l3vpnTtb.setMplsBos(true);
+ }
+ if (requireVlanPopBeforeMplsPush()) {
+ l3vpnTtb.pushVlan().setVlanId(VlanId.vlanId(VlanId.RESERVED));
+ }
+
+ GroupBucket l3vpnGrpBkt =
+ DefaultGroupBucket.createIndirectGroupBucket(l3vpnTtb.build());
+ int l3vpnIndex = getNextAvailableIndex();
+ int l3vpnGroupId = MPLS_L3VPN_SUBTYPE | (SUBTYPE_MASK & l3vpnIndex);
+ GroupKey l3vpnGroupKey = new DefaultGroupKey(
+ appKryo.serialize(l3vpnIndex));
+ GroupDescription l3vpnGroupDesc =
+ new DefaultGroupDescription(
+ deviceId,
+ GroupDescription.Type.INDIRECT,
+ new GroupBuckets(Collections.singletonList(l3vpnGrpBkt)),
+ l3vpnGroupKey,
+ l3vpnGroupId,
+ nextObj.appId());
+ GroupChainElem l3vpnGce = new GroupChainElem(l3vpnGroupDesc,
+ 1,
+ false,
+ deviceId);
+ updatePendingGroups(onelabelGroupInfo.nextGroupDesc().appCookie(), l3vpnGce);
+
+ gKeyChain.addFirst(onelabelGroupInfo.innerMostGroupDesc().appCookie());
+ gKeyChain.addFirst(onelabelGroupInfo.nextGroupDesc().appCookie());
+ gKeyChain.addFirst(l3vpnGroupKey);
+
+ //now we can replace the outerGrpDesc with the one we just created
+ onelabelGroupInfo.nextGroupDesc(l3vpnGroupDesc);
+
+ // we can't send the innermost group yet, as we have to create
+ // the dependent ECMP group first. So we store ...
+ unsentGroups.add(onelabelGroupInfo);
+
+ log.debug("Trying L3VPN: device:{} gid:{} group key:{} nextId:{}",
+ deviceId, Integer.toHexString(l3vpnGroupId),
+ l3vpnGroupKey, nextObj.id());
+
+ } else {
+ log.warn("Driver currently does not handle more than 1 MPLS "
+ + "labels. Not processing nextObjective {}", nextObj.id());
+ return;
+ }
+
+ // all groups in this chain
+ allGroupKeys.add(gKeyChain);
+ }
+ }
+
+ /**
+ * Processes the pseudo wire related next objective.
+ * This procedure try to reuse the mpls label groups,
+ * the mpls interface group and the l2 interface group.
+ *
+ * @param nextObjective the objective to process.
+ */
+ protected void processPwNextObjective(NextObjective nextObjective) {
+ log.warn("Pseudo wire extensions are not support for the OFDPA 2.0 {}", nextObjective.id());
+ }
+
+ //////////////////////////////////////
+ // Group Editing
+ //////////////////////////////////////
+ /**
+ * Adds a bucket to the top level group of a group-chain, and creates the chain.
+ * Ensures that bucket being added is not a duplicate, by checking existing
+ * buckets for the same output port.
+ *
+ * @param nextObjective the bucket information for a next group
+ * @param next the representation of the existing group-chain for this next objective
+ */
+ protected void addBucketToGroup(NextObjective nextObjective, NextGroup next) {
+ if (nextObjective.type() != NextObjective.Type.HASHED &&
+ nextObjective.type() != NextObjective.Type.BROADCAST) {
+ log.warn("AddBuckets not applied to nextType:{} in dev:{} for next:{}",
+ nextObjective.type(), deviceId, nextObjective.id());
+ fail(nextObjective, ObjectiveError.UNSUPPORTED);
+ return;
+ }
+
+ // first check to see if bucket being added is not a duplicate of an
+ // existing bucket. If it is for an existing output port, then its a
+ // duplicate.
+ Set<TrafficTreatment> duplicateBuckets = Sets.newHashSet();
+ List<Deque<GroupKey>> allActiveKeys = appKryo.deserialize(next.data());
+ Set<PortNumber> existingPorts = getExistingOutputPorts(allActiveKeys,
+ groupService,
+ deviceId);
+ Set<TrafficTreatment> nonDuplicateBuckets = Sets.newHashSet();
+ NextObjective objectiveToAdd;
+
+ nextObjective.next().forEach(trafficTreatment -> {
+ PortNumber portNumber = readOutPortFromTreatment(trafficTreatment);
+ if (portNumber == null) {
+ return;
+ }
+ if (existingPorts.contains(portNumber)) {
+ duplicateBuckets.add(trafficTreatment);
+ } else {
+ nonDuplicateBuckets.add(trafficTreatment);
+ }
+ });
+
+ if (duplicateBuckets.isEmpty()) {
+ // use the original objective
+ objectiveToAdd = nextObjective;
+ } else if (!nonDuplicateBuckets.isEmpty()) {
+ // only use the non-duplicate buckets if there are any
+ log.debug("Some buckets {} already exist in next id {}, duplicate "
+ + "buckets will be ignored.", duplicateBuckets, nextObjective.id());
+ // new next objective with non duplicate treatments
+ NextObjective.Builder builder = DefaultNextObjective.builder()
+ .withType(nextObjective.type())
+ .withId(nextObjective.id())
+ .withMeta(nextObjective.meta())
+ .fromApp(nextObjective.appId());
+ nonDuplicateBuckets.forEach(builder::addTreatment);
+
+ ObjectiveContext context = nextObjective.context().orElse(null);
+ objectiveToAdd = builder.addToExisting(context);
+ } else {
+ // buckets to add are already there - nothing to do
+ return;
+ }
+
+ if (nextObjective.type() == NextObjective.Type.HASHED) {
+ addBucketToHashGroup(objectiveToAdd, allActiveKeys);
+ } else if (nextObjective.type() == NextObjective.Type.BROADCAST) {
+ addBucketToBroadcastGroup(objectiveToAdd, allActiveKeys);
+ }
+ }
+
+ private void addBucketToHashGroup(NextObjective nextObjective,
+ List<Deque<GroupKey>> allActiveKeys) {
+ // storage for all group keys in the chain of groups created
+ List<Deque<GroupKey>> allGroupKeys = new ArrayList<>();
+ List<GroupInfo> unsentGroups = new ArrayList<>();
+ List<GroupBucket> newBuckets;
+ createHashBucketChains(nextObjective, allGroupKeys, unsentGroups);
+
+ // now we can create the buckets to add to the outermost L3 ECMP group
+ newBuckets = generateNextGroupBuckets(unsentGroups, SELECT);
+
+ // retrieve the original L3 ECMP group
+ Group l3ecmpGroup = retrieveTopLevelGroup(allActiveKeys, nextObjective.id());
+ if (l3ecmpGroup == null) {
+ fail(nextObjective, ObjectiveError.GROUPMISSING);
+ return;
+ }
+ GroupKey l3ecmpGroupKey = l3ecmpGroup.appCookie();
+ int l3ecmpGroupId = l3ecmpGroup.id().id();
+
+ // Although GroupDescriptions are not necessary for adding buckets to
+ // existing groups, we still use one in the GroupChainElem. When the latter is
+ // processed, the info will be extracted for the bucketAdd call to groupService
+ GroupDescription l3ecmpGroupDesc =
+ new DefaultGroupDescription(deviceId,
+ SELECT,
+ new GroupBuckets(newBuckets),
+ l3ecmpGroupKey,
+ l3ecmpGroupId,
+ nextObjective.appId());
+ GroupChainElem l3ecmpGce = new GroupChainElem(l3ecmpGroupDesc,
+ unsentGroups.size(),
+ true,
+ deviceId);
+
+ // update original NextGroup with new bucket-chain
+ // If active keys shows only the top-level group without a chain of groups,
+ // then it represents an empty group. Update by replacing empty chain.
+ Deque<GroupKey> newBucketChain = allGroupKeys.get(0);
+ newBucketChain.addFirst(l3ecmpGroupKey);
+ if (allActiveKeys.size() == 1 && allActiveKeys.get(0).size() == 1) {
+ allActiveKeys.clear();
+ }
+ allActiveKeys.add(newBucketChain);
+ updatePendingNextObjective(l3ecmpGroupKey, new OfdpaNextGroup(allActiveKeys, nextObjective));
+
+ log.debug("Adding to L3ECMP: device:{} gid:{} group key:{} nextId:{}",
+ deviceId, Integer.toHexString(l3ecmpGroupId),
+ l3ecmpGroupKey, nextObjective.id());
+
+ unsentGroups.forEach(groupInfo -> {
+ // send the innermost group
+ log.debug("Sending innermost group {} in group chain on device {} ",
+ Integer.toHexString(groupInfo.innerMostGroupDesc().givenGroupId()), deviceId);
+ updatePendingGroups(groupInfo.nextGroupDesc().appCookie(), l3ecmpGce);
+ groupService.addGroup(groupInfo.innerMostGroupDesc());
+ });
+ }
+
+ private void addBucketToBroadcastGroup(NextObjective nextObj,
+ List<Deque<GroupKey>> allActiveKeys) {
+ VlanId assignedVlan = readVlanFromSelector(nextObj.meta());
+ if (assignedVlan == null) {
+ log.warn("VLAN ID required by broadcast next obj is missing. "
+ + "Aborting add bucket to broadcast group for next:{} in dev:{}",
+ nextObj.id(), deviceId);
+ fail(nextObj, ObjectiveError.BADPARAMS);
+ return;
+ }
+ List<GroupInfo> groupInfos = prepareL2InterfaceGroup(nextObj, assignedVlan);
+ IpPrefix ipDst = readIpDstFromSelector(nextObj.meta());
+ if (ipDst != null) {
+ if (ipDst.isMulticast()) {
+ addBucketToL3MulticastGroup(nextObj, allActiveKeys, groupInfos, assignedVlan);
+ } else {
+ log.warn("Broadcast NextObj with non-multicast IP address {}", nextObj);
+ fail(nextObj, ObjectiveError.BADPARAMS);
+ }
+ } else {
+ addBucketToL2FloodGroup(nextObj, allActiveKeys, groupInfos, assignedVlan);
+ }
+ }
+
+ private void addBucketToL2FloodGroup(NextObjective nextObj,
+ List<Deque<GroupKey>> allActiveKeys,
+ List<GroupInfo> groupInfos,
+ VlanId assignedVlan) {
+ Group l2FloodGroup = retrieveTopLevelGroup(allActiveKeys, nextObj.id());
+
+ if (l2FloodGroup == null) {
+ log.warn("Can't find L2 flood group while adding bucket to it. NextObj = {}",
+ nextObj);
+ fail(nextObj, ObjectiveError.GROUPMISSING);
+ return;
+ }
+
+ GroupKey l2floodGroupKey = l2FloodGroup.appCookie();
+ int l2floodGroupId = l2FloodGroup.id().id();
+ List<GroupBucket> newBuckets = generateNextGroupBuckets(groupInfos, ALL);
+
+ GroupDescription l2FloodGroupDescription =
+ new DefaultGroupDescription(deviceId,
+ ALL,
+ new GroupBuckets(newBuckets),
+ l2floodGroupKey,
+ l2floodGroupId,
+ nextObj.appId());
+
+ GroupChainElem l2FloodGroupChainElement =
+ new GroupChainElem(l2FloodGroupDescription,
+ groupInfos.size(),
+ true,
+ deviceId);
+
+ updatePendingNextObjective(l2floodGroupKey,
+ new OfdpaNextGroup(allActiveKeys, nextObj));
+
+ //ensure assignedVlan applies to the chosen group
+ VlanId floodGroupVlan = extractVlanIdFromGroupId(l2floodGroupId);
+
+ if (!floodGroupVlan.equals(assignedVlan)) {
+ log.warn("VLAN ID {} does not match Flood group {} to which bucket is "
+ + "being added, for next:{} in dev:{}. Abort.", assignedVlan,
+ Integer.toHexString(l2floodGroupId), nextObj.id(), deviceId);
+ fail(nextObj, ObjectiveError.BADPARAMS);
+ return;
+ }
+
+ groupInfos.forEach(groupInfo -> {
+ // update original NextGroup with new bucket-chain
+ // If active keys shows only the top-level group without a chain of groups,
+ // then it represents an empty group. Update by replacing empty chain.
+ Deque<GroupKey> newBucketChain = new ArrayDeque<>();
+ newBucketChain.addFirst(groupInfo.nextGroupDesc().appCookie());
+ newBucketChain.addFirst(l2floodGroupKey);
+ if (allActiveKeys.size() == 1 && allActiveKeys.get(0).size() == 1) {
+ allActiveKeys.clear();
+ }
+ allActiveKeys.add(newBucketChain);
+
+ log.debug("Adding to L2FLOOD: device:{} gid:{} group key:{} nextId:{}",
+ deviceId, Integer.toHexString(l2floodGroupId),
+ l2floodGroupKey, nextObj.id());
+ // send the innermost group
+ log.debug("Sending innermost group {} in group chain on device {} ",
+ Integer.toHexString(groupInfo.innerMostGroupDesc().givenGroupId()),
+ deviceId);
+
+ updatePendingGroups(groupInfo.nextGroupDesc().appCookie(), l2FloodGroupChainElement);
+
+ DeviceId innerMostGroupDevice = groupInfo.innerMostGroupDesc().deviceId();
+ GroupKey innerMostGroupKey = groupInfo.innerMostGroupDesc().appCookie();
+ Group existsL2IGroup = groupService.getGroup(innerMostGroupDevice, innerMostGroupKey);
+
+ if (existsL2IGroup != null) {
+ // group already exist
+ processPendingAddGroupsOrNextObjs(innerMostGroupKey, true);
+ } else {
+ groupService.addGroup(groupInfo.innerMostGroupDesc());
+ }
+ });
+ }
+
+ private void addBucketToL3MulticastGroup(NextObjective nextObj,
+ List<Deque<GroupKey>> allActiveKeys,
+ List<GroupInfo> groupInfos,
+ VlanId assignedVlan) {
+ // create the buckets to add to the outermost L3 Multicast group
+ List<GroupBucket> newBuckets = Lists.newArrayList();
+ groupInfos.forEach(groupInfo -> {
+ // Points to L3 interface group if there is one.
+ // Otherwise points to L2 interface group directly.
+ GroupDescription nextGroupDesc = (groupInfo.nextGroupDesc() != null) ?
+ groupInfo.nextGroupDesc() : groupInfo.innerMostGroupDesc();
+ TrafficTreatment.Builder treatmentBuilder = DefaultTrafficTreatment.builder();
+ treatmentBuilder.group(new GroupId(nextGroupDesc.givenGroupId()));
+ GroupBucket newBucket = DefaultGroupBucket.createAllGroupBucket(treatmentBuilder.build());
+ newBuckets.add(newBucket);
+ });
+
+ // get the group being edited
+ Group l3mcastGroup = retrieveTopLevelGroup(allActiveKeys, nextObj.id());
+ if (l3mcastGroup == null) {
+ fail(nextObj, ObjectiveError.GROUPMISSING);
+ return;
+ }
+ GroupKey l3mcastGroupKey = l3mcastGroup.appCookie();
+ int l3mcastGroupId = l3mcastGroup.id().id();
+
+ //ensure assignedVlan applies to the chosen group
+ VlanId expectedVlan = extractVlanIdFromGroupId(l3mcastGroupId);
+ if (!expectedVlan.equals(assignedVlan)) {
+ log.warn("VLAN ID {} does not match L3 Mcast group {} to which bucket is "
+ + "being added, for next:{} in dev:{}. Abort.", assignedVlan,
+ Integer.toHexString(l3mcastGroupId), nextObj.id(), deviceId);
+ fail(nextObj, ObjectiveError.BADPARAMS);
+ }
+ GroupDescription l3mcastGroupDescription =
+ new DefaultGroupDescription(deviceId,
+ ALL,
+ new GroupBuckets(newBuckets),
+ l3mcastGroupKey,
+ l3mcastGroupId,
+ nextObj.appId());
+ GroupChainElem l3mcastGce = new GroupChainElem(l3mcastGroupDescription,
+ groupInfos.size(),
+ true,
+ deviceId);
+ groupInfos.forEach(groupInfo -> {
+ // update original NextGroup with new bucket-chain
+ Deque<GroupKey> newBucketChain = new ArrayDeque<>();
+ newBucketChain.addFirst(groupInfo.innerMostGroupDesc().appCookie());
+ // Add L3 interface group to the chain if there is one.
+ if (!groupInfo.nextGroupDesc().equals(groupInfo.innerMostGroupDesc())) {
+ newBucketChain.addFirst(groupInfo.nextGroupDesc().appCookie());
+ }
+ newBucketChain.addFirst(l3mcastGroupKey);
+ // If active keys shows only the top-level group without a chain of groups,
+ // then it represents an empty group. Update by replacing empty chain.
+ if (allActiveKeys.size() == 1 && allActiveKeys.get(0).size() == 1) {
+ allActiveKeys.clear();
+ }
+ allActiveKeys.add(newBucketChain);
+
+ updatePendingGroups(groupInfo.nextGroupDesc().appCookie(), l3mcastGce);
+ // Point next group to inner-most group, if any
+ if (!groupInfo.nextGroupDesc().equals(groupInfo.innerMostGroupDesc())) {
+ GroupChainElem innerGce = new GroupChainElem(groupInfo.nextGroupDesc(),
+ 1,
+ false,
+ deviceId);
+ updatePendingGroups(groupInfo.innerMostGroupDesc().appCookie(), innerGce);
+ }
+ log.debug("Adding to L3MCAST: device:{} gid:{} group key:{} nextId:{}",
+ deviceId, Integer.toHexString(l3mcastGroupId),
+ l3mcastGroupKey, nextObj.id());
+ // send the innermost group
+ log.debug("Sending innermost group {} in group chain on device {} ",
+ Integer.toHexString(groupInfo.innerMostGroupDesc().givenGroupId()),
+ deviceId);
+ groupService.addGroup(groupInfo.innerMostGroupDesc());
+
+ });
+
+ updatePendingNextObjective(l3mcastGroupKey,
+ new OfdpaNextGroup(allActiveKeys, nextObj));
+ }
+
+ /**
+ * Removes the bucket in the top level group of a possible group-chain. Does
+ * not remove the groups in the group-chain pointed to by this bucket, as they
+ * may be in use (referenced by other groups) elsewhere.
+ *
+ * @param nextObjective the bucket information for a next group
+ * @param next the representation of the existing group-chain for this next objective
+ */
+ protected void removeBucketFromGroup(NextObjective nextObjective, NextGroup next) {
+ if (nextObjective.type() != NextObjective.Type.HASHED &&
+ nextObjective.type() != NextObjective.Type.BROADCAST) {
+ log.warn("RemoveBuckets not applied to nextType:{} in dev:{} for next:{}",
+ nextObjective.type(), deviceId, nextObjective.id());
+ fail(nextObjective, ObjectiveError.UNSUPPORTED);
+ return;
+ }
+ Set<PortNumber> portsToRemove = Sets.newHashSet();
+ Collection<TrafficTreatment> treatments = nextObjective.next();
+ for (TrafficTreatment treatment : treatments) {
+ // find the bucket to remove by noting the outport, and figuring out the
+ // top-level group in the group-chain that indirectly references the port
+ PortNumber portToRemove = readOutPortFromTreatment(treatment);
+ if (portToRemove == null) {
+ log.warn("treatment {} of next objective {} has no outport.. cannot remove bucket"
+ + "from group in dev: {}", treatment, nextObjective.id(), deviceId);
+ } else {
+ portsToRemove.add(portToRemove);
+ }
+ }
+
+ if (portsToRemove.isEmpty()) {
+ log.warn("next objective {} has no outport.. cannot remove bucket"
+ + "from group in dev: {}", nextObjective.id(), deviceId);
+ fail(nextObjective, ObjectiveError.BADPARAMS);
+ }
+
+ List<Deque<GroupKey>> allActiveKeys = appKryo.deserialize(next.data());
+ List<Deque<GroupKey>> chainsToRemove = Lists.newArrayList();
+ for (Deque<GroupKey> gkeys : allActiveKeys) {
+ // last group in group chain should have a single bucket pointing to port
+
+ GroupKey groupWithPort = gkeys.peekLast();
+ Group group = groupService.getGroup(deviceId, groupWithPort);
+ if (group == null) {
+ log.warn("Inconsistent group chain found when removing bucket"
+ + "for next:{} in dev:{}", nextObjective.id(), deviceId);
+ continue;
+ }
+ if (group.buckets().buckets().isEmpty()) {
+ log.warn("Can't get output port information from group {} " +
+ "because there is no bucket in the group.",
+ group.id().toString());
+ continue;
+ }
+ PortNumber pout = readOutPortFromTreatment(
+ group.buckets().buckets().get(0).treatment());
+ if (portsToRemove.contains(pout)) {
+ chainsToRemove.add(gkeys);
+ }
+ }
+ if (chainsToRemove.isEmpty()) {
+ log.warn("Could not find appropriate group-chain for removing bucket"
+ + " for next id {} in dev:{}", nextObjective.id(), deviceId);
+ fail(nextObjective, ObjectiveError.BADPARAMS);
+ return;
+ }
+ List<GroupBucket> bucketsToRemove = Lists.newArrayList();
+ //first group key is the one we want to modify
+ GroupKey modGroupKey = chainsToRemove.get(0).peekFirst();
+ Group modGroup = groupService.getGroup(deviceId, modGroupKey);
+ for (Deque<GroupKey> foundChain : chainsToRemove) {
+ //second group key is the one we wish to remove the reference to
+ if (foundChain.size() < 2) {
+ // additional check to make sure second group key exist in
+ // the chain.
+ log.warn("Can't find second group key from chain {}",
+ foundChain);
+ continue;
+ }
+ GroupKey pointedGroupKey = foundChain.stream().collect(Collectors.toList()).get(1);
+ Group pointedGroup = groupService.getGroup(deviceId, pointedGroupKey);
+
+ if (pointedGroup == null) {
+ continue;
+ }
+
+ GroupBucket bucket;
+ if (nextObjective.type() == NextObjective.Type.HASHED) {
+ bucket = DefaultGroupBucket.createSelectGroupBucket(
+ DefaultTrafficTreatment.builder()
+ .group(pointedGroup.id())
+ .build());
+ } else {
+ bucket = DefaultGroupBucket.createAllGroupBucket(
+ DefaultTrafficTreatment.builder()
+ .group(pointedGroup.id())
+ .build());
+ }
+
+ bucketsToRemove.add(bucket);
+ }
+
+ GroupBuckets removeBuckets = new GroupBuckets(bucketsToRemove);
+ List<String> pointedGroupIds; // for debug log
+ pointedGroupIds = bucketsToRemove.stream()
+ .map(GroupBucket::treatment)
+ .map(TrafficTreatment::allInstructions)
+ .flatMap(List::stream)
+ .filter(inst -> inst instanceof Instructions.GroupInstruction)
+ .map(inst -> (Instructions.GroupInstruction) inst)
+ .map(Instructions.GroupInstruction::groupId)
+ .map(GroupId::id)
+ .map(Integer::toHexString)
+ .map(id -> HEX_PREFIX + id)
+ .collect(Collectors.toList());
+
+ log.debug("Removing buckets from group id 0x{} pointing to group id(s) {} "
+ + "for next id {} in device {}", Integer.toHexString(modGroup.id().id()),
+ pointedGroupIds, nextObjective.id(), deviceId);
+ addPendingUpdateNextObjective(modGroupKey, nextObjective);
+ groupService.removeBucketsFromGroup(deviceId, modGroupKey,
+ removeBuckets, modGroupKey,
+ nextObjective.appId());
+ // update store
+ allActiveKeys.removeAll(chainsToRemove);
+ // If no buckets in the group, then retain an entry for the
+ // top level group which still exists.
+ if (allActiveKeys.isEmpty()) {
+ ArrayDeque<GroupKey> top = new ArrayDeque<>();
+ top.add(modGroupKey);
+ allActiveKeys.add(top);
+ }
+ flowObjectiveStore.putNextGroup(nextObjective.id(),
+ new OfdpaNextGroup(allActiveKeys, nextObjective));
+ }
+
+ /**
+ * Removes all groups in multiple possible group-chains that represent the next-obj.
+ *
+ * @param nextObjective the next objective to remove
+ * @param next the NextGroup that represents the existing group-chain for
+ * this next objective
+ */
+ protected void removeGroup(NextObjective nextObjective, NextGroup next) {
+ List<Deque<GroupKey>> allActiveKeys = appKryo.deserialize(next.data());
+
+ List<GroupKey> groupKeys = allActiveKeys.stream()
+ .map(Deque::getFirst).collect(Collectors.toList());
+ addPendingRemoveNextObjective(nextObjective, groupKeys);
+
+ allActiveKeys.forEach(groupChain -> groupChain.forEach(groupKey ->
+ groupService.removeGroup(deviceId, groupKey, nextObjective.appId())));
+ flowObjectiveStore.removeNextGroup(nextObjective.id());
+ }
+
+ protected void updatePendingNextObjective(GroupKey groupKey, OfdpaNextGroup nextGrp) {
+ pendingAddNextObjectives.asMap().compute(groupKey, (k, val) -> {
+ if (val == null) {
+ val = new CopyOnWriteArrayList<>();
+ }
+ val.add(nextGrp);
+ return val;
+ });
+ }
+
+ protected void updatePendingGroups(GroupKey groupKey, GroupChainElem gce) {
+ pendingGroups.asMap().compute(groupKey, (k, val) -> {
+ if (val == null) {
+ val = Sets.newConcurrentHashSet();
+ }
+ val.add(gce);
+ return val;
+ });
+ }
+
+ protected void addPendingUpdateNextObjective(GroupKey groupKey,
+ NextObjective nextObjective) {
+ pendingUpdateNextObjectives.compute(groupKey, (gKey, nextObjs) -> {
+ if (nextObjs != null) {
+ nextObjs.add(nextObjective);
+ } else {
+ nextObjs = Sets.newHashSet(nextObjective);
+ }
+ return nextObjs;
+ });
+ }
+
+ private void processPendingUpdateNextObjs(GroupKey groupKey) {
+ pendingUpdateNextObjectives.compute(groupKey, (gKey, nextObjs) -> {
+ if (nextObjs != null) {
+
+ nextObjs.forEach(nextObj -> {
+ log.debug("Group {} updated, update pending next objective {}.",
+ groupKey, nextObj);
+
+ pass(nextObj);
+ });
+ }
+ return Sets.newHashSet();
+ });
+ }
+
+ private void processPendingRemoveNextObjs(GroupKey key) {
+ pendingRemoveNextObjectives.asMap().forEach((nextObjective, groupKeys) -> {
+ if (groupKeys.isEmpty()) {
+ pendingRemoveNextObjectives.invalidate(nextObjective);
+ pass(nextObjective);
+ } else {
+ groupKeys.remove(key);
+ }
+ });
+ }
+
+ protected int getNextAvailableIndex() {
+ return (int) nextIndex.incrementAndGet();
+ }
+
+ protected Group retrieveTopLevelGroup(List<Deque<GroupKey>> allActiveKeys,
+ int nextid) {
+ GroupKey topLevelGroupKey;
+ if (!allActiveKeys.isEmpty()) {
+ topLevelGroupKey = allActiveKeys.get(0).peekFirst();
+ } else {
+ log.warn("Could not determine top level group while processing"
+ + "next:{} in dev:{}", nextid, deviceId);
+ return null;
+ }
+ Group topGroup = groupService.getGroup(deviceId, topLevelGroupKey);
+ if (topGroup == null) {
+ log.warn("Could not find top level group while processing "
+ + "next:{} in dev:{}", nextid, deviceId);
+ }
+ return topGroup;
+ }
+
+ protected void processPendingAddGroupsOrNextObjs(GroupKey key, boolean added) {
+ //first check for group chain
+ Set<OfdpaGroupHandlerUtility.GroupChainElem> gceSet = pendingGroups.asMap().remove(key);
+ if (gceSet != null) {
+ for (GroupChainElem gce : gceSet) {
+ log.debug("Group service {} group key {} in device {}. "
+ + "Processing next group in group chain with group id 0x{}",
+ (added) ? "ADDED" : "processed",
+ key, deviceId,
+ Integer.toHexString(gce.groupDescription().givenGroupId()));
+ processGroupChain(gce);
+ }
+ } else {
+ // otherwise chain complete - check for waiting nextObjectives
+ List<OfdpaGroupHandlerUtility.OfdpaNextGroup> nextGrpList =
+ pendingAddNextObjectives.getIfPresent(key);
+ if (nextGrpList != null) {
+ pendingAddNextObjectives.invalidate(key);
+ nextGrpList.forEach(nextGrp -> {
+ log.debug("Group service {} group key {} in device:{}. "
+ + "Done implementing next objective: {} <<-->> gid:0x{}",
+ (added) ? "ADDED" : "processed",
+ key, deviceId, nextGrp.nextObjective().id(),
+ Integer.toHexString(groupService.getGroup(deviceId, key)
+ .givenGroupId()));
+ pass(nextGrp.nextObjective());
+ flowObjectiveStore.putNextGroup(nextGrp.nextObjective().id(), nextGrp);
+
+ // check if addBuckets waiting for this completion
+ pendingBuckets.compute(nextGrp.nextObjective().id(), (nextId, pendBkts) -> {
+ if (pendBkts != null) {
+ pendBkts.forEach(pendBkt -> addBucketToGroup(pendBkt, nextGrp));
+ }
+ return null;
+ });
+ });
+ }
+ }
+ }
+
+ /**
+ * Processes next element of a group chain. Assumption is that if this
+ * group points to another group, the latter has already been created
+ * and this driver has received notification for it. A second assumption is
+ * that if there is another group waiting for this group then the appropriate
+ * stores already have the information to act upon the notification for the
+ * creation of this group.
+ * <p>
+ * The processing of the GroupChainElement depends on the number of groups
+ * this element is waiting on. For all group types other than SIMPLE, a
+ * GroupChainElement could be waiting on multiple groups.
+ *
+ * @param gce the group chain element to be processed next
+ */
+ private void processGroupChain(GroupChainElem gce) {
+ int waitOnGroups = gce.decrementAndGetGroupsWaitedOn();
+ if (waitOnGroups != 0) {
+ log.debug("GCE: {} not ready to be processed", gce);
+ return;
+ }
+ log.debug("GCE: {} ready to be processed", gce);
+ if (gce.addBucketToGroup()) {
+ groupService.addBucketsToGroup(gce.groupDescription().deviceId(),
+ gce.groupDescription().appCookie(),
+ gce.groupDescription().buckets(),
+ gce.groupDescription().appCookie(),
+ gce.groupDescription().appId());
+ } else {
+ groupService.addGroup(gce.groupDescription());
+ }
+ }
+
+ protected void addPendingRemoveNextObjective(NextObjective nextObjective,
+ List<GroupKey> groupKeys) {
+ pendingRemoveNextObjectives.put(nextObjective, groupKeys);
+ }
+
+ private class InnerGroupListener implements GroupListener {
+ @Override
+ public void event(GroupEvent event) {
+ log.trace("received group event of type {}", event.type());
+ switch (event.type()) {
+ case GROUP_ADDED:
+ processPendingAddGroupsOrNextObjs(event.subject().appCookie(), true);
+ break;
+ case GROUP_REMOVED:
+ processPendingRemoveNextObjs(event.subject().appCookie());
+ break;
+ case GROUP_UPDATED:
+ processPendingUpdateNextObjs(event.subject().appCookie());
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}