drivers/barefoot/src/main/java/org/onosproject/drivers/barefoot/TofinoPipelineProgrammable.java

/*
 * Copyright 2017-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.onosproject.drivers.barefoot;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TMultiplexedProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.onlab.util.SharedExecutors;
import org.onosproject.drivers.barefoot.pal.InvalidPalOperation;
import org.onosproject.drivers.barefoot.pal.pal;
import org.onosproject.drivers.barefoot.pal.pal_fec_type_t;
import org.onosproject.drivers.barefoot.pal.pal_port_speed_t;
import org.onosproject.incubator.net.config.basics.PortDescriptionsConfig;
import org.onosproject.net.AnnotationKeys;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Port;
import org.onosproject.net.config.NetworkConfigService;
import org.onosproject.net.config.basics.BasicDeviceConfig;
import org.onosproject.net.device.DeviceDescription;
import org.onosproject.net.device.DeviceDescriptionDiscovery;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.device.PortDescription;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.driver.Driver;
import org.onosproject.net.driver.DriverService;
import org.onosproject.net.pi.model.PiPipeconf;
import org.onosproject.net.pi.model.PiPipeconf.ExtensionType;
import org.onosproject.net.pi.model.PiPipelineProgrammable;
import org.onosproject.p4runtime.api.P4RuntimeClient;
import org.onosproject.p4runtime.api.P4RuntimeController;
import org.onosproject.provider.general.device.api.GeneralProviderDeviceConfig;
import org.slf4j.Logger;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;

import static com.google.common.base.Strings.isNullOrEmpty;
import static org.slf4j.LoggerFactory.getLogger;

/**
 * Implementation of the PiPipelineProgrammable for BMv2.
 */
public class TofinoPipelineProgrammable extends AbstractHandlerBehaviour implements PiPipelineProgrammable {

    // FIXME: default pipeconf should depend on the system. Maybe pass it via netcfg?
    private static final PiPipeconf DEFAULT_PIPECONF = TofinoDefaultPipeconfFactory.getMavericks();
    private static final String P4RUNTIME = "p4runtime";
    private static final String PROTOCOL_KEY_IP = "ip";
    private static final String DEVICE_ID = "deviceId";
    private static final int PAL_MGMT_PORT = 9090;
    private static final String PAL_THRIFT_SERVICE = "pal";

    private final Logger log = getLogger(getClass());

    @Override
    public CompletableFuture<Boolean> deployPipeconf(PiPipeconf pipeconf) {

        CompletableFuture<Boolean> result = new CompletableFuture<>();

        SharedExecutors.getPoolThreadExecutor().submit(() -> result.complete(doDeployConfig(pipeconf)));

        return result;
    }

    private boolean doDeployConfig(PiPipeconf pipeconf) {

        P4RuntimeController controller = handler().get(P4RuntimeController.class);

        DeviceId deviceId = handler().data().deviceId();

        if (!controller.hasClient(deviceId)) {
            log.warn("Unable to find client for {}, aborting pipeconf deploy", deviceId);
            return false;
        }

        P4RuntimeClient client = controller.getClient(deviceId);

        //creating the ByteBuffer with all the needed elements to set the pipeline on the device
        ByteBuffer pipelineBuffer = createPipelineBuffer(pipeconf,
                ImmutableList.of(ExtensionType.TOFINO_BIN, ExtensionType.TOFINO_CONTEXT_JSON));

        try {
            if (!client.setPipelineConfig(pipeconf, pipelineBuffer).get()) {
                log.warn("Unable to deploy pipeconf {} to {}", pipeconf.id(), deviceId);
                return false;
            }

            // It would be more logical to have this performed at device handshake, but P4runtime would reject any
            // command if a P4info has not been set first.
            if (!client.initStreamChannel().get()) {
                log.warn("Unable to init stream channel to {}.", deviceId);
                return false;
            }

            SharedExecutors.getPoolThreadExecutor().submit(() -> setupPorts(deviceId));

        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }

        return true;
    }

    // FIXME Currently use thrift, may use gNMI to manage it in the future.
    private void setupPorts(DeviceId deviceId) {
        NetworkConfigService netcfgService = handler().get(NetworkConfigService.class);
        GeneralProviderDeviceConfig providerConfig =
                netcfgService.getConfig(deviceId, GeneralProviderDeviceConfig.class);
        PortDescriptionsConfig portConfig =
                netcfgService.getConfig(deviceId, PortDescriptionsConfig.class);

        Map<String, String> values =
                providerConfig.protocolsInfo().get(P4RUNTIME).configValues();
        String ip = values.get(PROTOCOL_KEY_IP);
        byte tofinoDeviceId = Byte.parseByte(values.get(DEVICE_ID));

        List<PortDescription> ports = portConfig.portDescriptions();
        ports.forEach(port -> {
            pal_port_speed_t speed = getPortSpeed(port.portSpeed());
            addAndEnablePort(ip, tofinoDeviceId, (int) port.portNumber().toLong(), speed);
        });
    }

    private pal_port_speed_t getPortSpeed(long speed) {
        // Mbps -> BF_SPEED (1G~100G)
        speed = speed / 1000;

        if (speed >= 100) {
            return pal_port_speed_t.BF_SPEED_100G;
        }
        if (speed >= 50) {
            return pal_port_speed_t.BF_SPEED_50G;
        }
        if (speed >= 40) {
            return pal_port_speed_t.BF_SPEED_40G;
        }
        if (speed >= 25) {
            return pal_port_speed_t.BF_SPEED_25G;
        }
        if (speed >= 10) {
            return pal_port_speed_t.BF_SPEED_10G;
        }

        return pal_port_speed_t.BF_SPEED_1G;
    }

    private void addAndEnablePort(String ip, byte deviceId, int dp, pal_port_speed_t speed) {
        TTransport transport = null;
        try {
            transport = new TSocket(ip, PAL_MGMT_PORT);
            transport.open();
            TProtocol protocol = new TBinaryProtocol(transport);
            TMultiplexedProtocol mProtocol = new TMultiplexedProtocol(protocol,
                                                                      PAL_THRIFT_SERVICE);
            pal.Client client = new pal.Client(mProtocol);

            client.pal_port_add(deviceId, dp, speed, pal_fec_type_t.BF_FEC_TYP_NONE);
            client.pal_port_enable(deviceId, dp);
            transport.close();
        } catch (TException x) {
            log.error("Error adding port {} to device {} ({})", dp, deviceId, ip, x);
        } finally {
            if (transport != null) {
                transport.close();
            }
        }
    }

    private ByteBuffer createPipelineBuffer(PiPipeconf pipeconf, List<ExtensionType> targetConfigExtTypes) {
        if (targetConfigExtTypes == null || targetConfigExtTypes.isEmpty() || isNullOrEmpty(pipeconf.id().toString())) {

            log.warn("Not enough information to deploy the Pipeconf {} on the switch {}, {}", pipeconf.id(),
                    targetConfigExtTypes);
        } else {
            List<ByteBuffer> buffers = Lists.newLinkedList();
            //Name of the pipeconf.
            //Appears to be an arbitrary name and unrelated to p4 program
            String name = pipeconf.id().toString();
            buffers.add(ByteBuffer.allocate(4 + name.length())
                                           .order(ByteOrder.LITTLE_ENDIAN)
                                           .putInt(name.length())
                                           .put(name.getBytes(StandardCharsets.UTF_8)));

            // Build buffers for all the extensions needed to deploy the pipeconf to Tofino
            targetConfigExtTypes.forEach(targetConfigExtType -> {
                if (!pipeconf.extension(targetConfigExtType).isPresent()) {
                    // FIXME this will break the expected data format; the resulting buffer will be invalid.
                    // FIXME Consider a stronger response here, like an exception.
                    log.warn("Missing extension {} in pipeconf {}", targetConfigExtType, pipeconf.id());
                }
                InputStream targetConfig = pipeconf.extension(targetConfigExtType).get();
                try {
                    log.info("Setting extension {} in pipeconf {}", targetConfigExtType, pipeconf.id());
                    byte[] bin = IOUtils.toByteArray(targetConfig);
                    //length and byte of every extension
                    buffers.add(ByteBuffer.allocate(4 + bin.length)
                                          .order(ByteOrder.LITTLE_ENDIAN)
                                          .putInt(bin.length)
                                          .put(bin));
                } catch (IOException ex) {
                    // FIXME this will break the expected data format; the resulting buffer will be invalid.
                    // FIXME Consider a stronger response here, like an exception.
                    log.warn("Unable to load target-specific config for {}", ex.getMessage());
                }
            });

            // Merge the buffers
            int len = buffers.stream().mapToInt(Buffer::limit).sum();
            ByteBuffer deviceData = ByteBuffer.allocate(len);
            for (ByteBuffer b : buffers) {
                deviceData.put((ByteBuffer) b.flip());
            }
            deviceData.flip(); // prepare for reading
            return deviceData.asReadOnlyBuffer();
        }
        return null;
    }

    @Override
    public Optional<PiPipeconf> getDefaultPipeconf() {
        return Optional.of(DEFAULT_PIPECONF);
    }
}