protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/Controller.java

/*
 * Copyright 2015-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.onosproject.openflow.controller.impl;

import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.epoll.EpollServerSocketChannel;
import io.netty.channel.group.ChannelGroup;
import io.netty.channel.group.DefaultChannelGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.util.concurrent.GlobalEventExecutor;
import org.onlab.util.ItemNotFoundException;
import org.onosproject.net.DeviceId;
import org.onosproject.net.config.NetworkConfigRegistry;
import org.onosproject.net.driver.DefaultDriverData;
import org.onosproject.net.driver.DefaultDriverHandler;
import org.onosproject.net.driver.Driver;
import org.onosproject.net.driver.DriverService;
import org.onosproject.openflow.config.OpenFlowDeviceConfig;
import org.onosproject.openflow.controller.Dpid;
import org.onosproject.openflow.controller.driver.OpenFlowAgent;
import org.onosproject.openflow.controller.driver.OpenFlowSwitchDriver;
import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
import org.projectfloodlight.openflow.protocol.OFVersion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.security.DigestInputStream;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.UnrecoverableKeyException;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Dictionary;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.onlab.util.Tools.get;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.net.DeviceId.deviceId;
import static org.onosproject.openflow.controller.Dpid.uri;


/**
 * The main controller class.  Handles all setup and network listeners
 */
public class Controller {

    private static final Logger log = LoggerFactory.getLogger(Controller.class);

    private static final short MIN_KS_LENGTH = 6;

    //Default queues settings
    private static final short DEFAULT_QUEUE_SIZE = 5000;
    private static final short FIRST_QUEUE_SIZE = 1000;
    private static final short DEFAULT_BULK_SIZE = 100;
    private static final short DEFAULT_QUEUE_ID = 7;

    protected HashMap<String, String> controllerNodeIPsCache;

    private ChannelGroup cg;

    // Configuration options
    protected List<Integer> openFlowPorts = ImmutableList.of(6633, 6653);
    protected int workerThreads = 0;
    protected int[] cfgQueueSizes = {FIRST_QUEUE_SIZE, 0, 0, 0, 0, 0, 0, DEFAULT_QUEUE_SIZE};
    protected int[] cfgBulkSizes = new int[8];

    // Start time of the controller
    protected long systemStartTime;

    private OpenFlowAgent agent;

    private EventLoopGroup bossGroup;
    private EventLoopGroup workerGroup;

    enum TlsMode {
        DISABLED, // TLS is not used for OpenFlow connections
        ENABLED,  // Clients are required use TLS and present a client certificate
        STRICT,   // Clients must use TLS, and certificate must match the one specified in netcfg
    }
    private static final EnumSet<TlsMode> TLS_ENABLED = EnumSet.of(TlsMode.ENABLED, TlsMode.STRICT);

    protected TlsParams tlsParams;
    protected SSLContext sslContext;
    protected KeyStore keyStore;

    // Perf. related configuration
    protected static final int SEND_BUFFER_SIZE = 4 * 1024 * 1024;

    private DriverService driverService;
    private NetworkConfigRegistry netCfgService;

    public Controller() {
        Arrays.fill(cfgBulkSizes, DEFAULT_BULK_SIZE);
    }

    public int getQueueSize(int queueId) {
        return cfgQueueSizes[queueId];
    }

    public int getBulkSize(int queueId) {
        return cfgBulkSizes[queueId];
    }

    // **************
    // Initialization
    // **************

    private void addListeningPorts(Collection<Integer> ports) {
        if (cg == null) {
            return;
        }
        final ServerBootstrap bootstrap = createServerBootStrap();
        bootstrap.option(ChannelOption.SO_REUSEADDR, true);
        bootstrap.childOption(ChannelOption.SO_KEEPALIVE, true);
        bootstrap.childOption(ChannelOption.TCP_NODELAY, true);
        bootstrap.childOption(ChannelOption.SO_SNDBUF, Controller.SEND_BUFFER_SIZE);
//            bootstrap.childOption(ChannelOption.WRITE_BUFFER_WATER_MARK,
//                                  new WriteBufferWaterMark(8 * 1024, 32 * 1024));

        bootstrap.childHandler(new OFChannelInitializer(this, null, sslContext));

        Set<Integer> existingPorts = cg.stream()
                .map(Channel::localAddress)
                .filter(InetSocketAddress.class::isInstance)
                .map(InetSocketAddress.class::cast)
                .map(InetSocketAddress::getPort)
                .collect(Collectors.toSet());
        ports.removeAll(existingPorts);

        ports.forEach(port -> {
            // TODO revisit if this is best way to listen to multiple ports
            cg.add(bootstrap.bind(port).syncUninterruptibly().channel());
            log.info("Listening for OF switch connections on {}", port);
        });
    }

    private void removeListeningPorts(Collection<Integer> ports) {
        if (cg == null) {
            return;
        }
        Iterator<Channel> itr = cg.iterator();
        while (itr.hasNext()) {
            Channel c = itr.next();
            SocketAddress addr = c.localAddress();
            if (addr instanceof InetSocketAddress) {
                InetSocketAddress inetAddr = (InetSocketAddress) addr;
                Integer port = inetAddr.getPort();
                if (ports.contains(port)) {
                    log.info("No longer listening for OF switch connections on {}", port);
                    c.close();
                    itr.remove();
                }
            }

        }
    }

    private ServerBootstrap createServerBootStrap() {

        int bossThreads = Math.max(1, openFlowPorts.size());
        try {
            bossGroup = new EpollEventLoopGroup(bossThreads, groupedThreads("onos/of", "boss-%d", log));
            workerGroup = new EpollEventLoopGroup(workerThreads, groupedThreads("onos/of", "worker-%d", log));
            ServerBootstrap bs = new ServerBootstrap()
                    .group(bossGroup, workerGroup)
                    .channel(EpollServerSocketChannel.class);
            log.info("Using Epoll transport");
            return bs;
        } catch (Throwable e) {
            log.debug("Failed to initialize native (epoll) transport: {}", e.getMessage());
        }

// Requires 4.1.11 or later
//        try {
//            bossGroup = new KQueueEventLoopGroup(bossThreads, groupedThreads("onos/of", "boss-%d", log));
//            workerGroup = new KQueueEventLoopGroup(workerThreads, groupedThreads("onos/of", "worker-%d", log));
//            ServerBootstrap bs = new ServerBootstrap()
//                    .group(bossGroup, workerGroup)
//                    .channel(KQueueServerSocketChannel.class);
//            log.info("Using Kqueue transport");
//            return bs;
//        } catch (Throwable e) {
//            log.debug("Failed to initialize native (kqueue) transport. ", e.getMessage());
//        }

        bossGroup = new NioEventLoopGroup(bossThreads, groupedThreads("onos/of", "boss-%d", log));
        workerGroup = new NioEventLoopGroup(workerThreads, groupedThreads("onos/of", "worker-%d", log));
        log.info("Using Nio transport");
        return new ServerBootstrap()
                    .group(bossGroup, workerGroup)
                    .channel(NioServerSocketChannel.class);
    }

    public void setQueueParams(Dictionary<?, ?> properties, String sizeParamName, String bulkParamName, int queueId) {
        String queueSize = get(properties, sizeParamName);
        if (!Strings.isNullOrEmpty(queueSize)) {
            int size = Integer.parseInt(queueSize);
            if (size > 0) {
                this.cfgQueueSizes[queueId] = size;
            } else {
                throw new IllegalArgumentException(
                    String.format("%s value must be either a positive integer value", sizeParamName));
            }
        }
        String bulkSize = get(properties, bulkParamName);
        if (!Strings.isNullOrEmpty(bulkSize)) {
            int bulk = Integer.parseInt(bulkSize);
            if (bulk > 0) {
                this.cfgBulkSizes[queueId] = bulk;
            } else {
                throw new IllegalArgumentException(
                    String.format("%s value must be either a positive integer value", bulkParamName));
            }
        }
    }

    public void setConfigParams(Dictionary<?, ?> properties) {
        boolean restartRequired = setOpenFlowPorts(properties);
        restartRequired |= setWorkerThreads(properties);
        restartRequired |= setTlsParameters(properties);
        if (restartRequired) {
            restart();
        }
    }

    /**
     * Gets the list of listening ports from property dict.
     *
     * @param properties dictionary
     * @return true if restart is required
     */
    private boolean setWorkerThreads(Dictionary<?, ?> properties) {
        List<Integer> oldPorts = this.openFlowPorts;
        String ports = get(properties, "openflowPorts");
        List<Integer> newPorts = Collections.emptyList();
        if (!Strings.isNullOrEmpty(ports)) {
            newPorts = Stream.of(ports.split(","))
                    .map(s -> Integer.parseInt(s))
                    .collect(Collectors.toList());
        }

        Set<Integer> portsToAdd = Sets.newHashSet(newPorts);
        portsToAdd.removeAll(oldPorts);
        addListeningPorts(portsToAdd);

        Set<Integer> portsToRemove = Sets.newHashSet(oldPorts);
        portsToRemove.removeAll(newPorts);
        removeListeningPorts(portsToRemove);

        this.openFlowPorts = newPorts;
        log.debug("OpenFlow ports set to {}", this.openFlowPorts);
        return false; // restart is never required
    }

    /**
     * Gets the number of worker threads from property dict.
     *
     * @param properties dictionary
     * @return true if restart is required
     */
    private boolean setOpenFlowPorts(Dictionary<?, ?> properties) {
        Integer oldValue = this.workerThreads;

        String threads = get(properties, "workerThreads");
        if (!Strings.isNullOrEmpty(threads)) {
            this.workerThreads = Integer.parseInt(threads);
        }
        log.debug("Number of worker threads set to {}", this.workerThreads);

        setQueueParams(properties, "defaultQueueSize", "defaultBulkSize", DEFAULT_QUEUE_ID);
        setQueueParams(properties, "queueSizeN0", "bulkSizeN0", 0);
        setQueueParams(properties, "queueSizeN1", "bulkSizeN1", 1);
        setQueueParams(properties, "queueSizeN2", "bulkSizeN2", 2);
        setQueueParams(properties, "queueSizeN3", "bulkSizeN3", 3);
        setQueueParams(properties, "queueSizeN4", "bulkSizeN4", 4);
        setQueueParams(properties, "queueSizeN5", "bulkSizeN5", 5);
        setQueueParams(properties, "queueSizeN6", "bulkSizeN6", 6);

        return oldValue != this.workerThreads; // restart if number of threads has changed
    }

    static class TlsParams {
        final TlsMode mode;
        final String ksLocation;
        final String tsLocation;
        final String ksPwd;
        final String tsPwd;
        final byte[] ksSignature;
        final byte[] tsSignature;

        TlsParams(TlsMode mode, String ksLocation, String tsLocation,
                  String ksPwd, String tsPwd) {
            this.mode = mode;
            this.ksLocation = ksLocation;
            this.tsLocation = tsLocation;
            this.ksPwd = ksPwd;
            this.tsPwd = tsPwd;
            this.ksSignature = getSha1Checksum(ksLocation);
            this.tsSignature = getSha1Checksum(tsLocation);
        }

        public char[] ksPwd() {
            return ksPwd.toCharArray();
        }

        public char[] tsPwd() {
            return tsPwd.toCharArray();
        }

        public boolean isTlsEnabled() {
            return TLS_ENABLED.contains(mode);
        }

        public byte[] getSha1Checksum(String filepath) {
            if (filepath == null) {
                return new byte[0];
            }
            try {
                MessageDigest digest = MessageDigest.getInstance("SHA1");
                File f = new File(filepath);
                FileInputStream is = new FileInputStream(f);
                DigestInputStream dis = new DigestInputStream(is, digest);
                byte[] buffer = new byte[1024];
                while (dis.read(buffer) > 0) {
                    // nothing to do :)
                }
                is.close();
                return dis.getMessageDigest().digest();
            } catch (NoSuchAlgorithmException ignored) {
            } catch (IOException e) {
                log.info("Error reading file file: {}", filepath);
            }
            return new byte[0];
        }

        @Override
        public int hashCode() {
            if (mode == TlsMode.DISABLED) {
                return Objects.hash(mode);
            }
            return Objects.hash(mode, ksLocation, tsLocation,
                    ksPwd, tsPwd,
                    Arrays.hashCode(ksSignature),
                    Arrays.hashCode(tsSignature));
        }

        @Override
        public boolean equals(Object obj) {
            if (this == obj) {
                return true;
            }
            if (obj instanceof TlsParams) {
                final TlsParams that = (TlsParams) obj;
                if (this.getClass() != that.getClass()) {
                    return false;
                } else if (this.mode == that.mode && this.mode == TlsMode.DISABLED) {
                    // All disabled objects should be equal regardless of other params
                    return true;
                }
                return this.mode == that.mode &&
                        Objects.equals(this.ksLocation, that.ksLocation) &&
                        Objects.equals(this.tsLocation, that.tsLocation) &&
                        Objects.equals(this.ksPwd, that.ksPwd) &&
                        Objects.equals(this.tsPwd, that.tsPwd) &&
                        Arrays.equals(this.ksSignature, that.ksSignature) &&
                        Arrays.equals(this.tsSignature, that.tsSignature);
            }
            return false;
        }

        @Override
        public String toString() {
            return MoreObjects.toStringHelper(this)
                    .add("tlsMode", mode.toString().toLowerCase())
                    .add("ksLocation", ksLocation)
                    .add("tsLocation", tsLocation)
                    .toString();
        }
    }

    /**
     * Gets the TLS parameters from the properties dict, but fallback to the old approach of
     * system properties if the parameters are missing from the dict.
     *
     * @param properties dictionary
     * @return true if restart is required
     */
    private boolean setTlsParameters(Dictionary<?, ?> properties) {
        TlsParams oldParams = this.tlsParams;

        TlsMode mode = null;
        String tlsString = get(properties, "tlsMode");
        if (!Strings.isNullOrEmpty(tlsString)) {
            try {
                mode = TlsMode.valueOf(tlsString.toUpperCase());
            } catch (IllegalArgumentException e) {
                log.info("Invalid TLS mode {}. TLS is disabled.", tlsString);
                mode = TlsMode.DISABLED;
            }
        } else {
            // Fallback to system properties
            // TODO this method of configuring TLS is deprecated and should be removed eventually
            tlsString = System.getProperty("enableOFTLS");
            mode = !Strings.isNullOrEmpty(tlsString) && Boolean.parseBoolean(tlsString) ?
                    TlsMode.ENABLED : TlsMode.DISABLED;
        }

        String ksLocation = null, tsLocation = null, ksPwd = null, tsPwd = null;
        if (TLS_ENABLED.contains(mode)) {
            ksLocation = get(properties, "keyStore");
            if (Strings.isNullOrEmpty(ksLocation)) {
                // Fallback to system properties
                // TODO remove this eventually
                ksLocation = System.getProperty("javax.net.ssl.keyStore");
            }
            if (Strings.isNullOrEmpty(ksLocation)) {
                mode = TlsMode.DISABLED;
            }

            tsLocation = get(properties, "trustStore");
            if (Strings.isNullOrEmpty(tsLocation)) {
                // Fallback to system properties
                // TODO remove this eventually
                tsLocation = System.getProperty("javax.net.ssl.trustStore");
            }
            if (Strings.isNullOrEmpty(tsLocation)) {
                mode = TlsMode.DISABLED;
            }

            ksPwd = get(properties, "keyStorePassword");
            if (Strings.isNullOrEmpty(ksPwd)) {
                // Fallback to system properties
                // TODO remove this eventually
                ksPwd = System.getProperty("javax.net.ssl.keyStorePassword");
            }
            if (Strings.isNullOrEmpty(ksPwd) || MIN_KS_LENGTH > ksPwd.length()) {
                mode = TlsMode.DISABLED;
            }

            tsPwd = get(properties, "trustStorePassword");
            if (Strings.isNullOrEmpty(tsPwd)) {
                // Fallback to system properties
                // TODO remove this eventually
                tsPwd = System.getProperty("javax.net.ssl.trustStorePassword");
            }
            if (Strings.isNullOrEmpty(tsPwd) || MIN_KS_LENGTH > tsPwd.length()) {
                mode = TlsMode.DISABLED;
            }
        }
        this.tlsParams = new TlsParams(mode, ksLocation, tsLocation, ksPwd, tsPwd);
        log.info("OpenFlow TLS Params: {}", tlsParams);
        return !Objects.equals(this.tlsParams, oldParams); // restart if TLS params change
    }

    /**
     * Initialize internal data structures.
     */
    public void init() {
        // These data structures are initialized here because other
        // module's startUp() might be called before ours
        this.controllerNodeIPsCache = new HashMap<>();

        this.systemStartTime = System.currentTimeMillis();

        cg = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE);

        if (tlsParams.isTlsEnabled()) {
            initSsl();
        }
    }

    private void initSsl() {
        try {
            TrustManagerFactory tmFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
            KeyStore ts = KeyStore.getInstance("JKS");
            ts.load(new FileInputStream(tlsParams.tsLocation), tlsParams.tsPwd());
            tmFactory.init(ts);

            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
            keyStore = KeyStore.getInstance("JKS");
            keyStore.load(new FileInputStream(tlsParams.ksLocation), tlsParams.ksPwd());
            kmf.init(keyStore, tlsParams.ksPwd());

            sslContext = SSLContext.getInstance("TLS");
            sslContext.init(kmf.getKeyManagers(), tmFactory.getTrustManagers(), null);
        } catch (NoSuchAlgorithmException | KeyStoreException | CertificateException |
                IOException | KeyManagementException | UnrecoverableKeyException ex) {
            log.error("SSL init failed: {}", ex.getMessage());
        }
    }

    // **************
    // Utility methods
    // **************

    public Map<String, Long> getMemory() {
        Map<String, Long> m = new HashMap<>();
        Runtime runtime = Runtime.getRuntime();
        m.put("total", runtime.totalMemory());
        m.put("free", runtime.freeMemory());
        return m;
    }


    public Long getSystemUptime() {
        RuntimeMXBean rb = ManagementFactory.getRuntimeMXBean();
        return rb.getUptime();
    }

    public long getSystemStartTime() {
        return (this.systemStartTime);
    }

    public boolean isValidCertificate(Long dpid, Certificate peerCert) {
        if (!tlsParams.isTlsEnabled()) {
            return true;
        }

        if (netCfgService == null) {
            // netcfg service not available; accept any cert if not in strict mode
            return tlsParams.mode == TlsMode.ENABLED;
        }

        DeviceId deviceId = DeviceId.deviceId(Dpid.uri(new Dpid(dpid)));
        OpenFlowDeviceConfig config =
                netCfgService.getConfig(deviceId, OpenFlowDeviceConfig.class);
        if (config == null) {
            // Config not set for device, accept any cert if not in strict mode
            return tlsParams.mode == TlsMode.ENABLED;
        }

        Optional<String> alias = config.keyAlias();
        if (!alias.isPresent()) {
            // Config for device does not specify a certificate chain, accept any cert if not in strict mode
            return tlsParams.mode == TlsMode.ENABLED;
        }

        try {
            Certificate configuredCert = keyStore.getCertificate(alias.get());
            //TODO there's probably a better way to compare these
            return Objects.deepEquals(peerCert, configuredCert);
        } catch (KeyStoreException e) {
            log.info("failed to load key", e);
        }
        return false;
    }

    /**
     * Forward to the driver-manager to get an IOFSwitch instance.
     *
     * @param dpid data path id
     * @param desc switch description
     * @param ofv  OpenFlow version
     * @return switch instance
     */
    protected OpenFlowSwitchDriver getOFSwitchInstance(long dpid,
                                                       OFDescStatsReply desc,
                                                       OFVersion ofv) {
        Dpid dpidObj = new Dpid(dpid);

        Driver driver;
        try {
            driver = driverService.getDriver(DeviceId.deviceId(Dpid.uri(dpidObj)));
        } catch (ItemNotFoundException e) {
            driver = driverService.getDriver(desc.getMfrDesc(), desc.getHwDesc(), desc.getSwDesc());
        }

        if (driver == null) {
            log.error("No OpenFlow driver for {} : {}", dpidObj, desc);
            return null;
        }

        log.info("Driver '{}' assigned to device {}", driver.name(), dpidObj);

        if (!driver.hasBehaviour(OpenFlowSwitchDriver.class)) {
            log.error("Driver {} does not support OpenFlowSwitchDriver behaviour", driver.name());
            return null;
        }

        DefaultDriverHandler handler =
                new DefaultDriverHandler(new DefaultDriverData(driver, deviceId(uri(dpidObj))));
        OpenFlowSwitchDriver ofSwitchDriver =
                driver.createBehaviour(handler, OpenFlowSwitchDriver.class);
        ofSwitchDriver.init(dpidObj, desc, ofv);
        ofSwitchDriver.setAgent(agent);
        ofSwitchDriver.setRoleHandler(new RoleManager(ofSwitchDriver));
        return ofSwitchDriver;
    }

    public void start(OpenFlowAgent ag, DriverService driverService,
                      NetworkConfigRegistry netCfgService) {
        log.info("Starting OpenFlow IO");
        this.agent = ag;
        this.driverService = driverService;
        this.netCfgService = netCfgService;
        this.init();
        this.addListeningPorts(this.openFlowPorts);
    }


    public void stop() {
        log.info("Stopping OpenFlow IO");
        if (cg != null) {
            cg.close();
            cg = null;
        }

        // Shut down all event loops to terminate all threads.
        bossGroup.shutdownGracefully();
        workerGroup.shutdownGracefully();

        // Wait until all threads are terminated.
        try {
            bossGroup.terminationFuture().sync();
            workerGroup.terminationFuture().sync();
        } catch (InterruptedException e) {
            log.warn("Interrupted while stopping", e);
            Thread.currentThread().interrupt();
        }
    }

    private void restart() {
        // only restart if we are already running
        if (cg != null) {
            stop();
            start(this.agent, this.driverService, this.netCfgService);
        }
    }

}