drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeGroupProgrammable.java

/*
 * Copyright 2017-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.onosproject.drivers.p4runtime;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.onlab.util.KryoNamespace;
import org.onosproject.core.GroupId;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.instructions.Instruction;
import org.onosproject.net.flow.instructions.PiInstruction;
import org.onosproject.net.group.Group;
import org.onosproject.net.group.GroupBucket;
import org.onosproject.net.group.GroupOperation;
import org.onosproject.net.group.GroupOperations;
import org.onosproject.net.group.GroupProgrammable;
import org.onosproject.net.group.GroupStore;
import org.onosproject.net.pi.model.PiPipelineInterpreter;
import org.onosproject.net.pi.runtime.PiActionProfileId;
import org.onosproject.net.pi.runtime.PiAction;
import org.onosproject.net.pi.runtime.PiActionGroup;
import org.onosproject.net.pi.runtime.PiActionGroupId;
import org.onosproject.net.pi.runtime.PiActionGroupMember;
import org.onosproject.net.pi.runtime.PiActionGroupMemberId;
import org.onosproject.net.pi.runtime.PiTableAction;
import org.onosproject.net.pi.runtime.PiTableId;
import org.onosproject.p4runtime.api.P4RuntimeClient;
import org.onosproject.p4runtime.api.P4RuntimeGroupReference;
import org.onosproject.p4runtime.api.P4RuntimeGroupWrapper;
import org.onosproject.store.serializers.KryoNamespaces;
import org.slf4j.Logger;

import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import static org.slf4j.LoggerFactory.getLogger;

/**
 * Implementation of the group programmable behaviour for P4Runtime.
 */
public class P4RuntimeGroupProgrammable extends AbstractP4RuntimeHandlerBehaviour implements GroupProgrammable {
    private static final String ACT_GRP_MEMS = "action group members";
    private static final String DELETE = "delete";
    private static final String ACT_GRP = "action group";
    private static final String INSERT = "insert";
    private static final Logger log = getLogger(P4RuntimeGroupProgrammable.class);
    private static final int GROUP_ID_MASK = 0xffff;
    public static final KryoNamespace KRYO = new KryoNamespace.Builder()
            .register(KryoNamespaces.API)
            .register(DefaultP4RuntimeGroupCookie.class)
            .build("P4RuntimeGroupProgrammable");

    /*
     * About action groups in P4runtime:
     * The type field is a place holder in p4runtime.proto right now, and we haven't defined it yet. You can assume all
     * the groups are "select" as per the OF spec. As a remainder, in the P4 terminology a member corresponds to an OF
     * bucket. Each member can also be used directly in the match table (kind of like an OF indirect group).
     */

    // TODO: make this attribute configurable by child drivers (e.g. BMv2 or Tofino)
    /*
    When updating an existing rule, if true, we issue a DELETE operation before inserting the new one, otherwise we
    issue a MODIFY operation. This is useful fore devices that do not support MODIFY operations for table entries.
     */
    private boolean deleteBeforeUpdate = true;

    // TODO: can remove this check as soon as the multi-apply-per-same-flow rule bug is fixed.
    /*
    If true, we ignore re-installing rules that are already known in the ENTRY_STORE, i.e. same match key and action.
     */
    private boolean checkStoreBeforeUpdate = true;

    // Needed to synchronize operations over the same group.
    private static final Map<P4RuntimeGroupReference, Lock> GROUP_LOCKS = Maps.newConcurrentMap();

    // TODO: replace with distribute store
    private static final Map<P4RuntimeGroupReference, P4RuntimeGroupWrapper> GROUP_STORE = Maps.newConcurrentMap();

    private PiPipelineInterpreter interpreter;

    protected boolean init() {
        if (!setupBehaviour()) {
            return false;
        }
        Device device = deviceService.getDevice(deviceId);
        // Need an interpreter to map the bucket treatment to a PI action
        if (!device.is(PiPipelineInterpreter.class)) {
            log.warn("Can't find interpreter for device {}", device.id());
        } else {
            interpreter = device.as(PiPipelineInterpreter.class);
        }
        return true;
    }

    @Override
    public void performGroupOperation(DeviceId deviceId, GroupOperations groupOps) {
        if (!init()) {
            // Ignore group operation of not initialized.
            return;
        }
        Device device = handler().get(DeviceService.class).getDevice(deviceId);

        for (GroupOperation groupOp : groupOps.operations()) {
            processGroupOp(device, groupOp);
        }
    }

    private void processGroupOp(Device device, GroupOperation groupOp) {
        GroupId groupId = groupOp.groupId();
        GroupStore groupStore = handler().get(GroupStore.class);
        Group group = groupStore.getGroup(device.id(), groupId);

        // Most of this logic can go in a core service, e.g. PiGroupTranslationService
        // From a P4Runtime perspective, we need first to insert members, then the group.
        PiActionGroupId piActionGroupId = PiActionGroupId.of(groupOp.groupId().id());

        PiActionGroup.Builder piActionGroupBuilder = PiActionGroup.builder()
                .withId(piActionGroupId);

        switch (group.type()) {
            case SELECT:
                piActionGroupBuilder.withType(PiActionGroup.Type.SELECT);
                break;
            default:
                log.warn("Group type {} not supported, ignore group {}.", group.type(), groupId);
                return;
        }
        /*
            Problem:
            In P4Runtime, action profiles (i.e. group tables) are specific to one or more tables.
            Mapping of treatments depends on the target table. How do we derive the target table from here?

            Solution:
            - Add table information into app cookie and put into group description
         */
        // TODO: notify group service if we get deserialize error
        DefaultP4RuntimeGroupCookie defaultP4RuntimeGroupCookie = KRYO.deserialize(group.appCookie().key());
        PiTableId piTableId = defaultP4RuntimeGroupCookie.tableId();
        PiActionProfileId piActionProfileId = defaultP4RuntimeGroupCookie.actionProfileId();
        piActionGroupBuilder.withActionProfileId(piActionProfileId);

        List<PiActionGroupMember> members = buildMembers(group, piActionGroupId, piTableId);
        if (members == null) {
            log.warn("Can't build members for group {} on {}", group, device.id());
            return;
        }

        piActionGroupBuilder.addMembers(members);
        PiActionGroup piActionGroup = piActionGroupBuilder.build();

        P4RuntimeGroupReference groupRef =
                new P4RuntimeGroupReference(deviceId, piActionProfileId, piActionGroupId);
        Lock lock = GROUP_LOCKS.computeIfAbsent(groupRef, k -> new ReentrantLock());
        lock.lock();


        try {
            P4RuntimeGroupWrapper oldGroupWrapper = GROUP_STORE.get(groupRef);
            P4RuntimeGroupWrapper newGroupWrapper =
                    new P4RuntimeGroupWrapper(piActionGroup, group, System.currentTimeMillis());
            boolean success;
            switch (groupOp.opType()) {
                case ADD:
                case MODIFY:
                    success = writeGroupToDevice(oldGroupWrapper, piActionGroup, members);
                    if (success) {
                        GROUP_STORE.put(groupRef, newGroupWrapper);
                    }
                    break;
                case DELETE:
                    success = deleteGroupFromDevice(piActionGroup, members);
                    if (success) {
                        GROUP_STORE.remove(groupRef);
                    }
                    break;
                default:
                    throw new UnsupportedOperationException();
            }
        } finally {
            lock.unlock();
        }
    }

    /**
     * Installs action group and members to device via client interface.
     *
     * @param oldGroupWrapper old group wrapper for the group; null if not exists
     * @param piActionGroup the action group to be installed
     * @param members members of the action group
     * @return true if install success; false otherwise
     */
    private boolean writeGroupToDevice(P4RuntimeGroupWrapper oldGroupWrapper,
                                       PiActionGroup piActionGroup,
                                       Collection<PiActionGroupMember> members) {
        boolean success = true;
        CompletableFuture<Boolean> writeSuccess;
        if (checkStoreBeforeUpdate && oldGroupWrapper != null &&
                oldGroupWrapper.piActionGroup().equals(piActionGroup)) {
            // Action group already exists, ignore it
            return true;
        }
        if (deleteBeforeUpdate && oldGroupWrapper != null) {
            success = deleteGroupFromDevice(oldGroupWrapper.piActionGroup(),
                                            oldGroupWrapper.piActionGroup().members());
        }
        writeSuccess = client.writeActionGroupMembers(piActionGroup,
                                                      members,
                                                      P4RuntimeClient.WriteOperationType.INSERT,
                                                      pipeconf);
        success = success && completeSuccess(writeSuccess, ACT_GRP_MEMS, INSERT);

        writeSuccess = client.writeActionGroup(piActionGroup,
                                               P4RuntimeClient.WriteOperationType.INSERT,
                                               pipeconf);
        success = success && completeSuccess(writeSuccess, ACT_GRP, INSERT);
        return success;
    }

    private boolean deleteGroupFromDevice(PiActionGroup piActionGroup,
                                          Collection<PiActionGroupMember> members) {
        boolean success;
        CompletableFuture<Boolean> writeSuccess;
        writeSuccess = client.writeActionGroup(piActionGroup,
                                P4RuntimeClient.WriteOperationType.DELETE,
                                pipeconf);
        success = completeSuccess(writeSuccess, ACT_GRP, DELETE);
        writeSuccess = client.writeActionGroupMembers(piActionGroup,
                                       members,
                                       P4RuntimeClient.WriteOperationType.DELETE,
                                       pipeconf);
        success = success && completeSuccess(writeSuccess, ACT_GRP_MEMS, DELETE);
        return success;
    }

    private boolean completeSuccess(CompletableFuture<Boolean> completableFuture,
                                    String topic, String action) {
        try {
            return completableFuture.get();
        } catch (InterruptedException | ExecutionException e) {
            log.warn("Can't {} {} due to {}", action, topic, e.getMessage());
            return false;
        }
    }

    /**
     * Build pi action group members from group.
     *
     * @param group the group
     * @param piActionGroupId the PI action group id of the group
     * @param piTableId the PI table related to the group
     * @return list of PI action group members; null if can't build member list
     */
    private List<PiActionGroupMember> buildMembers(Group group, PiActionGroupId piActionGroupId, PiTableId piTableId) {
        GroupId groupId = group.id();
        ImmutableList.Builder<PiActionGroupMember> membersBuilder = ImmutableList.builder();

        int bucketIdx = 0;
        for (GroupBucket bucket : group.buckets().buckets()) {
            /*
            Problem:
            In P4Runtime action group members, i.e. action buckets, are associated to a numeric ID chosen
            at member insertion time. This ID must be unique for the whole action profile (i.e. the group table in
            OpenFlow). In ONOS, GroupBucket doesn't specify any ID.

            Solutions:
            - Change GroupBucket API to force application wanting to perform group operations to specify a member id.
            - Maintain state to dynamically allocate/deallocate member IDs, e.g. in a dedicated service, or in a
            P4Runtime Group Provider.

            Hack:
            Statically derive member ID by combining groupId and position of the bucket in the list.
             */
            ByteBuffer bb = ByteBuffer.allocate(4)
                    .putShort((short) (piActionGroupId.id() & GROUP_ID_MASK))
                    .putShort((short) bucketIdx);
            bb.rewind();
            int memberId = bb.getInt();

            bucketIdx++;
            PiAction action;
            if (interpreter != null) {
                // if we have interpreter, use interpreter
                try {
                    action = interpreter.mapTreatment(bucket.treatment(), piTableId);
                } catch (PiPipelineInterpreter.PiInterpreterException e) {
                    log.warn("Can't map treatment {} to action due to {}, ignore group {}",
                             bucket.treatment(), e.getMessage(), groupId);
                    return null;
                }
            } else {
                // if we don't have interpreter, accept PiInstruction only
                TrafficTreatment treatment = bucket.treatment();

                if (treatment.allInstructions().size() > 1) {
                    log.warn("Treatment {} has multiple instructions, ignore group {}",
                             treatment, groupId);
                    return null;
                }
                Instruction instruction = treatment.allInstructions().get(0);
                if (instruction.type() != Instruction.Type.PROTOCOL_INDEPENDENT) {
                    log.warn("Instruction {} is not a PROTOCOL_INDEPENDENT type, ignore group {}",
                             instruction, groupId);
                    return null;
                }

                PiInstruction piInstruction = (PiInstruction) instruction;
                if (piInstruction.action().type() != PiTableAction.Type.ACTION) {
                    log.warn("Action {} is not an ACTION type, ignore group {}",
                             piInstruction.action(), groupId);
                    return null;
                }
                action = (PiAction) piInstruction.action();
            }

            PiActionGroupMember member = PiActionGroupMember.builder()
                    .withId(PiActionGroupMemberId.of(memberId))
                    .withAction(action)
                    .withWeight(bucket.weight())
                    .build();

            membersBuilder.add(member);
        }
        return membersBuilder.build();
    }

    @Override
    public Collection<Group> getGroups() {
        if (!init()) {
            return Collections.emptySet();
        }

        Collection<Group> result = Sets.newHashSet();
        Collection<PiActionProfileId> piActionProfileIds = Sets.newHashSet();

        // Collection action profile Ids
        // TODO: find better way to get all action profile ids....
        GROUP_STORE.forEach((groupRef, wrapper) -> piActionProfileIds.add(groupRef.actionProfileId()));

        AtomicBoolean success = new AtomicBoolean(true);
        piActionProfileIds.forEach(actionProfileId -> {
            Collection<PiActionGroup> piActionGroups = Sets.newHashSet();
            try {
                Collection<PiActionGroup> groupsFromDevice =
                        client.dumpGroups(actionProfileId, pipeconf).get();
                if (groupsFromDevice == null) {
                    // Got error
                    success.set(false);
                } else {
                    piActionGroups.addAll(groupsFromDevice);
                }
            } catch (ExecutionException | InterruptedException e) {
                log.error("Exception while dumping groups for action profile {}: {}",
                          actionProfileId.id(), deviceId, e);
                success.set(false);
            }

            piActionGroups.forEach(piActionGroup -> {
                PiActionGroupId actionGroupId = piActionGroup.id();
                P4RuntimeGroupReference groupRef =
                        new P4RuntimeGroupReference(deviceId, actionProfileId, actionGroupId);
                P4RuntimeGroupWrapper wrapper = GROUP_STORE.get(groupRef);

                if (wrapper == null) {
                    // group exists in client, but can't find in ONOS
                    log.warn("Can't find action profile group {} from local store.",
                             groupRef);
                    return;
                }
                if (!wrapper.piActionGroup().equals(piActionGroup)) {
                    log.warn("Group from device is different to group from local store.");
                    return;
                }
                result.add(wrapper.group());

            });
        });

        if (!success.get()) {
            // Got error while dump groups from device.
            return Collections.emptySet();
        } else {
            return result;
        }
    }

    /**
     * P4Runtime app cookie for group.
     */
    public static class DefaultP4RuntimeGroupCookie {
        private PiTableId tableId;
        private PiActionProfileId piActionProfileId;
        private Integer groupId;

        public DefaultP4RuntimeGroupCookie(PiTableId tableId,
                                           PiActionProfileId piActionProfileId,
                                           Integer groupId) {
            this.tableId = tableId;
            this.piActionProfileId = piActionProfileId;
            this.groupId = groupId;
        }

        public PiTableId tableId() {
            return tableId;
        }

        public PiActionProfileId actionProfileId() {
            return piActionProfileId;
        }

        public Integer groupId() {
            return groupId;
        }
    }
}