apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java

/*
 * Copyright 2019-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.onosproject.k8snetworking.impl;

import org.onlab.packet.ARP;
import org.onlab.packet.Ethernet;
import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpPrefix;
import org.onlab.packet.MacAddress;
import org.onlab.packet.TpPort;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.k8snetworking.api.K8sFlowRuleService;
import org.onosproject.k8snetworking.api.K8sNetwork;
import org.onosproject.k8snetworking.api.K8sNetworkEvent;
import org.onosproject.k8snetworking.api.K8sNetworkListener;
import org.onosproject.k8snetworking.api.K8sNetworkService;
import org.onosproject.k8snetworking.api.K8sPort;
import org.onosproject.k8snetworking.util.RulePopulatorUtil;
import org.onosproject.k8snode.api.K8sHost;
import org.onosproject.k8snode.api.K8sHostService;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.mastership.MastershipService;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.DriverService;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.slf4j.Logger;

import java.util.Objects;
import java.util.concurrent.ExecutorService;

import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.k8snetworking.api.Constants.DEFAULT_GATEWAY_MAC;
import static org.onosproject.k8snetworking.api.Constants.EXT_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.POD_RESOLUTION_TABLE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_ROUTER_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTER_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
import static org.onosproject.k8snode.api.Constants.DEFAULT_EXTERNAL_GATEWAY_MAC;
import static org.onosproject.k8snode.api.K8sApiConfig.Mode.PASSTHROUGH;
import static org.slf4j.LoggerFactory.getLogger;

/**
 * Provides POD's internal to external connectivity using source NAT (SNAT).
 */
@Component(immediate = true)
public class K8sRoutingSnatHandler {

    private final Logger log = getLogger(getClass());

    private static final int HOST_PREFIX = 32;

    // we try to avoid port number overlapping with node port (30000 ~ 32767)
    // in case the user has customized node port range, the following static
    // value should be changed accordingly
    private static final int TP_PORT_MINIMUM_NUM = 32768;
    private static final int TP_PORT_MAXIMUM_NUM = 65535;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected CoreService coreService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected DeviceService deviceService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected DriverService driverService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected LeadershipService leadershipService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected MastershipService mastershipService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected ClusterService clusterService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected K8sNetworkService k8sNetworkService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected K8sNodeService k8sNodeService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected K8sHostService k8sHostService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected K8sFlowRuleService k8sFlowRuleService;

    private final InternalK8sNetworkListener k8sNetworkListener =
            new InternalK8sNetworkListener();
    private final InternalK8sNodeListener k8sNodeListener =
            new InternalK8sNodeListener();
    private final ExecutorService eventExecutor = newSingleThreadExecutor(
            groupedThreads(this.getClass().getSimpleName(), "event-handler", log));

    private ApplicationId appId;
    private NodeId localNodeId;

    @Activate
    protected void activate() {
        appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);

        localNodeId = clusterService.getLocalNode().id();
        leadershipService.runForLeadership(appId.name());
        k8sNetworkService.addListener(k8sNetworkListener);
        k8sNodeService.addListener(k8sNodeListener);

        log.info("Started");
    }

    @Deactivate
    protected void deactivate() {
        k8sNodeService.removeListener(k8sNodeListener);
        k8sNetworkService.removeListener(k8sNetworkListener);
        leadershipService.withdraw(appId.name());
        eventExecutor.shutdown();

        log.info("Stopped");
    }

    private void setContainerToExtRule(K8sNode k8sNode, boolean install) {

        K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());

        if (net == null) {
            return;
        }

        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchTunnelId(Long.valueOf(net.segmentId()))
                .matchEthDst(DEFAULT_GATEWAY_MAC);

        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
                .setOutput(k8sNode.intgToExtPatchPortNum());

        k8sFlowRuleService.setRule(
                appId,
                k8sNode.intgBridge(),
                sBuilder.build(),
                tBuilder.build(),
                PRIORITY_EXTERNAL_ROUTING_RULE,
                ROUTING_TABLE,
                install);
    }

    private void setExtToContainerRule(K8sNode k8sNode,
                                       K8sPort k8sPort, boolean install) {

        K8sNetwork net = k8sNetworkService.network(k8sPort.networkId());

        if (net == null) {
            return;
        }

        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchIPDst(IpPrefix.valueOf(k8sPort.ipAddress(), HOST_PREFIX));

        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
                .setOutput(k8sNode.extToIntgPatchPortNum());

        k8sFlowRuleService.setRule(
                appId,
                k8sNode.extBridge(),
                sBuilder.build(),
                tBuilder.build(),
                PRIORITY_STATEFUL_SNAT_RULE,
                POD_RESOLUTION_TABLE,
                install);
    }

    private void setExtSnatDownstreamRule(K8sNode k8sNode,
                                          boolean install) {
        DeviceId deviceId = k8sNode.extBridge();

        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), HOST_PREFIX));

        ExtensionTreatment natTreatment = RulePopulatorUtil
                .niciraConnTrackTreatmentBuilder(driverService, deviceId)
                .commit(false)
                .natAction(true)
                .table((short) POD_RESOLUTION_TABLE)
                .build();

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .setEthSrc(DEFAULT_GATEWAY_MAC)
                .extension(natTreatment, deviceId)
                .build();

        k8sFlowRuleService.setRule(
                appId,
                deviceId,
                sBuilder.build(),
                treatment,
                PRIORITY_STATEFUL_SNAT_RULE,
                EXT_ENTRY_TABLE,
                install);
    }

    private void setExtSnatUpstreamRule(K8sNode k8sNode,
                                        boolean install) {

        K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());

        if (net == null) {
            return;
        }

        TrafficSelector selector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchInPort(k8sNode.extToIntgPatchPortNum())
                .matchEthDst(DEFAULT_GATEWAY_MAC)
                .build();

        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();

        if (install) {
            ExtensionTreatment natTreatment = RulePopulatorUtil
                    .niciraConnTrackTreatmentBuilder(driverService, k8sNode.extBridge())
                    .commit(true)
                    .natFlag(CT_NAT_SRC_FLAG)
                    .natAction(true)
                    .natIp(k8sNode.extBridgeIp())
                    .natPortMin(TpPort.tpPort(TP_PORT_MINIMUM_NUM))
                    .natPortMax(TpPort.tpPort(TP_PORT_MAXIMUM_NUM))
                    .build();

            tBuilder.extension(natTreatment, k8sNode.extBridge())
                    .setEthSrc(k8sNode.extBridgeMac())
                    .setEthDst(k8sNode.extGatewayMac());

            if (k8sNode.mode() == PASSTHROUGH) {
                tBuilder.setOutput(k8sNode.extToRouterPortNum());
            } else {
                if (MacAddress.valueOf(DEFAULT_EXTERNAL_GATEWAY_MAC).equals(
                        k8sNode.extGatewayMac())) {
                    tBuilder.setOutput(k8sNode.extIntfPortNum());
                } else {
                    tBuilder.setOutput(k8sNode.extBridgePortNum());
                }
            }
        }

        k8sFlowRuleService.setRule(
                appId,
                k8sNode.extBridge(),
                selector,
                tBuilder.build(),
                PRIORITY_STATEFUL_SNAT_RULE,
                EXT_ENTRY_TABLE,
                install);
    }

    private void setExtIntfArpRule(K8sNode k8sNode, boolean install) {
        k8sNodeService.completeNodes().forEach(n -> {
            Device device = deviceService.getDevice(n.extBridge());
            TrafficSelector selector = DefaultTrafficSelector.builder()
                    .matchEthType(Ethernet.TYPE_ARP)
                    .matchArpOp(ARP.OP_REQUEST)
                    .matchArpTpa(Ip4Address.valueOf(k8sNode.extBridgeIp().toString()))
                    .build();

            TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                    .setArpOp(ARP.OP_REPLY)
                    .extension(buildMoveEthSrcToDstExtension(device), device.id())
                    .extension(buildMoveArpShaToThaExtension(device), device.id())
                    .extension(buildMoveArpSpaToTpaExtension(device), device.id())
                    .setEthSrc(k8sNode.extBridgeMac())
                    .setArpSha(k8sNode.extBridgeMac())
                    .setArpSpa(Ip4Address.valueOf(k8sNode.extBridgeIp().toString()))
                    .setOutput(PortNumber.IN_PORT)
                    .build();

            k8sFlowRuleService.setRule(
                    appId,
                    n.extBridge(),
                    selector,
                    treatment,
                    PRIORITY_STATEFUL_SNAT_RULE,
                    EXT_ENTRY_TABLE,
                    install);
        });
    }

    private void setRouterSnatUpstreamRule(K8sNode k8sNode,
                                           K8sRouterBridge bridge,
                                           boolean install) {
        if (k8sNode.routerPortNum() == null) {
            return;
        }

        TrafficSelector ipSelector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchInPort(k8sNode.routerToExtPortNum())
                .build();

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .setOutput(k8sNode.routerPortNum())
                .build();

        k8sFlowRuleService.setRule(
                appId,
                bridge.deviceId(),
                ipSelector,
                treatment,
                PRIORITY_ROUTER_RULE,
                ROUTER_ENTRY_TABLE,
                install);

        TrafficSelector arpSelector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_ARP)
                .matchInPort(k8sNode.routerToExtPortNum())
                .build();

        k8sFlowRuleService.setRule(
                appId,
                bridge.deviceId(),
                arpSelector,
                treatment,
                PRIORITY_ROUTER_RULE,
                ROUTER_ENTRY_TABLE,
                install);
    }

    private void setRouterSnatDownstreamRule(K8sNode k8sNode,
                                             K8sRouterBridge bridge,
                                             boolean install) {
        if (k8sNode.routerPortNum() == null) {
            return;
        }

        TrafficSelector ipSelector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchInPort(k8sNode.routerPortNum())
                .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), 32))
                .build();

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .setOutput(k8sNode.routerToExtPortNum())
                .build();

        k8sFlowRuleService.setRule(
                appId,
                bridge.deviceId(),
                ipSelector,
                treatment,
                PRIORITY_ROUTER_RULE,
                ROUTER_ENTRY_TABLE,
                install);

        TrafficSelector arpSelector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_ARP)
                .matchInPort(k8sNode.routerPortNum())
                .matchArpTpa(Ip4Address.valueOf(k8sNode.extBridgeIp().toString()))
                .build();

        k8sFlowRuleService.setRule(
                appId,
                bridge.deviceId(),
                arpSelector,
                treatment,
                PRIORITY_ROUTER_RULE,
                ROUTER_ENTRY_TABLE,
                install);
    }

    private void setRouterSnatRules(K8sNode k8sNode, boolean install) {
        for (K8sHost host : k8sHostService.completeHosts()) {
            if (host.nodeNames().contains(k8sNode.hostname())) {
                K8sRouterBridge bridge = host.routerBridges().stream()
                        .filter(b -> b.segmentId() == k8sNode.segmentId())
                        .findAny().orElse(null);
                if (bridge != null) {
                    setRouterSnatUpstreamRule(k8sNode, bridge, install);
                    setRouterSnatDownstreamRule(k8sNode, bridge, install);
                }
            }
        }
    }

    private class InternalK8sNodeListener implements K8sNodeListener {

        private boolean isRelevantHelper() {
            return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
        }

        @Override
        public void event(K8sNodeEvent event) {
            switch (event.type()) {
                case K8S_NODE_COMPLETE:
                    eventExecutor.execute(() -> processNodeCompletion(event.subject()));
                    break;
                case K8S_NODE_UPDATED:
                    eventExecutor.execute(() -> processNodeUpdate(event.subject()));
                    break;
                case K8S_NODE_OFF_BOARDED:
                    eventExecutor.execute(() -> processNodeOffboard(event.subject()));
                    break;
                case K8S_NODE_INCOMPLETE:
                default:
                    break;
            }
        }

        private void processNodeCompletion(K8sNode k8sNode) {
            if (!isRelevantHelper()) {
                return;
            }

            setExtIntfArpRule(k8sNode, true);
            setExtSnatDownstreamRule(k8sNode, true);
            setContainerToExtRule(k8sNode, true);
            setRouterSnatRules(k8sNode, true);
        }

        private void processNodeOffboard(K8sNode k8sNode) {
            if (!isRelevantHelper()) {
                return;
            }

            setExtIntfArpRule(k8sNode, false);
            setExtSnatDownstreamRule(k8sNode, false);
            setContainerToExtRule(k8sNode, false);
            setRouterSnatRules(k8sNode, false);
        }

        private void processNodeUpdate(K8sNode k8sNode) {
            if (k8sNode.extGatewayMac() != null) {
                setExtSnatUpstreamRule(k8sNode, true);
            }
        }
    }

    private class InternalK8sNetworkListener implements K8sNetworkListener {

        private boolean isRelevantHelper() {
            return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
        }

        @Override
        public void event(K8sNetworkEvent event) {
            switch (event.type()) {
                case K8S_PORT_ACTIVATED:
                    eventExecutor.execute(() -> processPortActivation(event.port()));
                    break;
                case K8S_PORT_REMOVED:
                    eventExecutor.execute(() -> processPortRemoval(event.port()));
                    break;
                default:
                    break;
            }
        }

        private void processPortActivation(K8sPort port) {
            if (!isRelevantHelper()) {
                return;
            }

            k8sNodeService.completeNodes().forEach(n ->
                    setExtToContainerRule(n, port, true));
        }

        private void processPortRemoval(K8sPort port) {
            if (!isRelevantHelper()) {
                return;
            }

            k8sNodeService.completeNodes().forEach(n ->
                    setExtToContainerRule(n, port, false));
        }
    }
}