apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackK8sIntegrationManager.java

/*
 * Copyright 2020-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.onosproject.openstacknetworking.impl;


import org.onlab.packet.ARP;
import org.onlab.packet.Ethernet;
import org.onlab.packet.IPv4;
import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpAddress;
import org.onlab.packet.IpPrefix;
import org.onlab.packet.MacAddress;
import org.onlab.packet.TpPort;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.net.Port;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.openstacknetworking.api.Constants;
import org.onosproject.openstacknetworking.api.InstancePort;
import org.onosproject.openstacknetworking.api.InstancePortService;
import org.onosproject.openstacknetworking.api.OpenstackFlowRuleService;
import org.onosproject.openstacknetworking.api.OpenstackGroupRuleService;
import org.onosproject.openstacknetworking.api.OpenstackK8sIntegrationService;
import org.onosproject.openstacknetworking.api.OpenstackNetwork;
import org.onosproject.openstacknetworking.api.OpenstackNetworkService;
import org.onosproject.openstacknode.api.OpenstackNode;
import org.onosproject.openstacknode.api.OpenstackNodeService;
import org.openstack4j.model.network.Network;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.slf4j.Logger;

import java.util.Map;
import java.util.Objects;

import static org.onosproject.net.AnnotationKeys.PORT_NAME;
import static org.onosproject.openstacknetworking.api.Constants.DHCP_TABLE;
import static org.onosproject.openstacknetworking.api.Constants.FLAT_TABLE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_CNI_PT_IP_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_CNI_PT_NODE_PORT_ARP_EXT_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_CNI_PT_NODE_PORT_ARP_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_CNI_PT_NODE_PORT_IP_RULE;
import static org.onosproject.openstacknetworking.api.Constants.STAT_FLAT_OUTBOUND_TABLE;
import static org.onosproject.openstacknetworking.api.OpenstackNetwork.Type.FLAT;
import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.shiftIpDomain;
import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.structurePortName;
import static org.onosproject.openstacknetworking.util.RulePopulatorUtil.buildPortRangeMatches;
import static org.onosproject.openstacknode.api.Constants.INTEGRATION_TO_PHYSICAL_PREFIX;
import static org.slf4j.LoggerFactory.getLogger;

/**
 * Implementation of openstack kubernetes integration service.
 */

@Component(
    immediate = true,
    service = { OpenstackK8sIntegrationService.class }
)
public class OpenstackK8sIntegrationManager implements OpenstackK8sIntegrationService {

    protected final Logger log = getLogger(getClass());

    private static final String SHIFTED_IP_PREFIX = "172.10";
    private static final int NODE_PORT_MIN = 30000;
    private static final int NODE_PORT_MAX = 32767;
    public static final String NODE_FAKE_IP_STR = "172.172.172.172";

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected CoreService coreService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected OpenstackFlowRuleService osFlowRuleService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected OpenstackGroupRuleService osGroupRuleService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected OpenstackNodeService osNodeService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected OpenstackNetworkService osNetworkService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected InstancePortService instancePortService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected DeviceService deviceService;

    private ApplicationId appId;

    @Activate
    protected void activate() {
        appId = coreService.registerApplication(Constants.OPENSTACK_NETWORKING_APP_ID);

        log.info("Started");
    }

    @Deactivate
    protected void deactivate() {
        log.info("Stopped");
    }

    @Override
    public void installCniPtNodeRules(IpAddress k8sNodeIp,
                                      IpPrefix podCidr, IpPrefix serviceCidr,
                                      IpAddress podGatewayIp, String osK8sIntPortName,
                                      MacAddress k8sIntOsPortMac) {
        setNodeToPodIpRules(k8sNodeIp, podCidr, serviceCidr,
                podGatewayIp, osK8sIntPortName, k8sIntOsPortMac, true);
        setPodToNodeIpRules(k8sNodeIp, podGatewayIp, osK8sIntPortName, true);
    }

    @Override
    public void uninstallCniPtNodeRules(IpAddress k8sNodeIp,
                                        IpPrefix podCidr, IpPrefix serviceCidr,
                                        IpAddress podGatewayIp, String osK8sIntPortName,
                                        MacAddress k8sIntOsPortMac) {
        setNodeToPodIpRules(k8sNodeIp, podCidr, serviceCidr,
                podGatewayIp, osK8sIntPortName, k8sIntOsPortMac, false);
        setPodToNodeIpRules(k8sNodeIp, podGatewayIp, osK8sIntPortName, false);
    }

    @Override
    public void installCniPtNodePortRules(IpAddress k8sNodeIp, String osK8sExtPortName) {
        setNodePortIngressRules(k8sNodeIp, osK8sExtPortName, true);
        setNodePortEgressRules(k8sNodeIp, osK8sExtPortName, true);

        setArpRequestRules(k8sNodeIp, osK8sExtPortName, true);
        setArpReplyRules(k8sNodeIp, osK8sExtPortName, true);
    }

    @Override
    public void uninstallCniPtNodePortRules(IpAddress k8sNodeIp, String osK8sExtPortName) {
        setNodePortIngressRules(k8sNodeIp, osK8sExtPortName, false);
        setNodePortEgressRules(k8sNodeIp, osK8sExtPortName, false);

        setArpRequestRules(k8sNodeIp, osK8sExtPortName, false);
        setArpReplyRules(k8sNodeIp, osK8sExtPortName, false);
    }

    private void setNodeToPodIpRules(IpAddress k8sNodeIp,
                                     IpPrefix podCidr, IpPrefix serviceCidr,
                                     IpAddress gatewayIp, String osK8sIntPortName,
                                     MacAddress k8sIntOsPortMac, boolean install) {

        OpenstackNode osNode = osNodeByNodeIp(k8sNodeIp);

        if (osNode == null) {
            return;
        }

        PortNumber osK8sIntPortNum = osNode.portNumByName(osK8sIntPortName);

        if (osK8sIntPortNum == null) {
            return;
        }

        TrafficSelector originalPodSelector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchIPSrc(IpPrefix.valueOf(k8sNodeIp, 32))
                .matchIPDst(podCidr)
                .build();

        TrafficSelector transformedPodSelector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchIPSrc(IpPrefix.valueOf(k8sNodeIp, 32))
                .matchIPDst(IpPrefix.valueOf(shiftIpDomain(podCidr.toString(), SHIFTED_IP_PREFIX)))
                .build();

        TrafficSelector serviceSelector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchIPSrc(IpPrefix.valueOf(k8sNodeIp, 32))
                .matchIPDst(serviceCidr)
                .build();

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .setIpSrc(gatewayIp)
                .setEthSrc(k8sIntOsPortMac)
                .setOutput(osK8sIntPortNum)
                .build();

        osFlowRuleService.setRule(
                appId,
                osNode.intgBridge(),
                originalPodSelector,
                treatment,
                PRIORITY_CNI_PT_IP_RULE,
                FLAT_TABLE,
                install
        );

        osFlowRuleService.setRule(
                appId,
                osNode.intgBridge(),
                transformedPodSelector,
                treatment,
                PRIORITY_CNI_PT_IP_RULE,
                FLAT_TABLE,
                install
        );

        osFlowRuleService.setRule(
                appId,
                osNode.intgBridge(),
                serviceSelector,
                treatment,
                PRIORITY_CNI_PT_IP_RULE,
                FLAT_TABLE,
                install
        );
    }

    private void setPodToNodeIpRules(IpAddress k8sNodeIp, IpAddress gatewayIp,
                                     String osK8sIntPortName, boolean install) {
        InstancePort instPort = instPortByNodeIp(k8sNodeIp);

        if (instPort == null) {
            return;
        }

        OpenstackNode osNode = osNodeByNodeIp(k8sNodeIp);

        if (osNode == null) {
            return;
        }

        PortNumber osK8sIntPortNum = osNode.portNumByName(osK8sIntPortName);

        if (osK8sIntPortNum == null) {
            return;
        }

        TrafficSelector selector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchIPDst(IpPrefix.valueOf(gatewayIp, 32))
                .matchInPort(osK8sIntPortNum)
                .build();

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .setIpDst(k8sNodeIp)
                .setEthDst(instPort.macAddress())
                .transition(STAT_FLAT_OUTBOUND_TABLE)
                .build();

        osFlowRuleService.setRule(
                appId,
                osNode.intgBridge(),
                selector,
                treatment,
                PRIORITY_CNI_PT_IP_RULE,
                DHCP_TABLE,
                install
        );
    }

    private void setNodePortIngressRules(IpAddress k8sNodeIp,
                                         String osK8sExtPortName,
                                         boolean install) {
        OpenstackNode osNode = osNodeByNodeIp(k8sNodeIp);

        if (osNode == null) {
            return;
        }

        PortNumber osK8sExtPortNum = portNumberByNodeIpAndPortName(k8sNodeIp, osK8sExtPortName);

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .setOutput(osK8sExtPortNum)
                .build();

        Map<TpPort, TpPort> portRangeMatchMap =
                buildPortRangeMatches(NODE_PORT_MIN, NODE_PORT_MAX);

        portRangeMatchMap.forEach((key, value) -> {
            TrafficSelector.Builder tcpSelectorBuilder = DefaultTrafficSelector.builder()
                    .matchEthType(Ethernet.TYPE_IPV4)
                    .matchIPDst(IpPrefix.valueOf(k8sNodeIp, 32))
                    .matchIPProtocol(IPv4.PROTOCOL_TCP)
                    .matchTcpDstMasked(key, value);

            TrafficSelector.Builder udpSelectorBuilder = DefaultTrafficSelector.builder()
                    .matchEthType(Ethernet.TYPE_IPV4)
                    .matchIPDst(IpPrefix.valueOf(k8sNodeIp, 32))
                    .matchIPProtocol(IPv4.PROTOCOL_UDP)
                    .matchUdpDstMasked(key, value);

            osFlowRuleService.setRule(
                    appId,
                    osNode.intgBridge(),
                    tcpSelectorBuilder.build(),
                    treatment,
                    PRIORITY_CNI_PT_NODE_PORT_IP_RULE,
                    FLAT_TABLE,
                    install
            );

            osFlowRuleService.setRule(
                    appId,
                    osNode.intgBridge(),
                    udpSelectorBuilder.build(),
                    treatment,
                    PRIORITY_CNI_PT_NODE_PORT_IP_RULE,
                    FLAT_TABLE,
                    install
            );
        });
    }

    private void setNodePortEgressRules(IpAddress k8sNodeIp,
                                        String osK8sExtPortName,
                                        boolean install) {
        InstancePort instPort = instPortByNodeIp(k8sNodeIp);

        if (instPort == null) {
            return;
        }

        OpenstackNode osNode = osNodeByNodeIp(k8sNodeIp);

        if (osNode == null) {
            return;
        }

        PortNumber osK8sExtPortNum = portNumberByNodeIpAndPortName(k8sNodeIp, osK8sExtPortName);

        Port phyPort = phyPortByInstPort(instPort);

        if (phyPort == null) {
            log.warn("No phys interface found for instance port {}", instPort);
            return;
        }

        TrafficSelector selector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_IPV4)
                .matchInPort(osK8sExtPortNum)
                .build();

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .setEthSrc(instPort.macAddress())
                .setOutput(phyPort.number())
                .build();

        osFlowRuleService.setRule(
                appId,
                osNode.intgBridge(),
                selector,
                treatment,
                PRIORITY_CNI_PT_NODE_PORT_IP_RULE,
                DHCP_TABLE,
                install
        );
    }

    private void setArpRequestRules(IpAddress k8sNodeIp, String osK8sExtPortName, boolean install) {
        InstancePort instPort = instPortByNodeIp(k8sNodeIp);

        if (instPort == null) {
            return;
        }

        OpenstackNode osNode = osNodeByNodeIp(k8sNodeIp);

        if (osNode == null) {
            return;
        }

        PortNumber osK8sExtPortNum = portNumberByNodeIpAndPortName(k8sNodeIp, osK8sExtPortName);

        TrafficSelector selector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_ARP)
                .matchArpOp(ARP.OP_REQUEST)
                .matchInPort(osK8sExtPortNum)
                .build();

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .transition(STAT_FLAT_OUTBOUND_TABLE)
                .build();

        osFlowRuleService.setRule(
                appId,
                osNode.intgBridge(),
                selector,
                treatment,
                PRIORITY_CNI_PT_NODE_PORT_ARP_RULE,
                DHCP_TABLE,
                install
        );

        Port phyPort = phyPortByInstPort(instPort);

        if (phyPort == null) {
            log.warn("No phys interface found for instance port {}", instPort);
            return;
        }

        TrafficTreatment extTreatment = DefaultTrafficTreatment.builder()
                .setOutput(phyPort.number())
                .build();

        osFlowRuleService.setRule(
                appId,
                osNode.intgBridge(),
                selector,
                extTreatment,
                PRIORITY_CNI_PT_NODE_PORT_ARP_EXT_RULE,
                FLAT_TABLE,
                install
        );
    }

    private void setArpReplyRules(IpAddress k8sNodeIp, String osK8sExtPortName, boolean install) {
        OpenstackNode osNode = osNodeByNodeIp(k8sNodeIp);

        if (osNode == null) {
            return;
        }

        PortNumber osK8sExtPortNum = portNumberByNodeIpAndPortName(k8sNodeIp, osK8sExtPortName);

        TrafficSelector selector = DefaultTrafficSelector.builder()
                .matchEthType(Ethernet.TYPE_ARP)
                .matchArpOp(ARP.OP_REPLY)
                .matchArpTpa(Ip4Address.valueOf(NODE_FAKE_IP_STR))
                .build();

        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                .setOutput(osK8sExtPortNum)
                .build();

        osFlowRuleService.setRule(
                appId,
                osNode.intgBridge(),
                selector,
                treatment,
                PRIORITY_CNI_PT_NODE_PORT_ARP_RULE,
                FLAT_TABLE,
                install
        );
    }

    private InstancePort instPortByNodeIp(IpAddress k8sNodeIp) {
        return instancePortService.instancePorts().stream().filter(p -> {
            OpenstackNetwork.Type netType = osNetworkService.networkType(p.networkId());
            return netType == FLAT && p.ipAddress().equals(k8sNodeIp);
        }).findAny().orElse(null);
    }

    private OpenstackNode osNodeByNodeIp(IpAddress k8sNodeIp) {
        InstancePort instPort = instPortByNodeIp(k8sNodeIp);

        if (instPort == null) {
            return null;
        }

        return osNodeService.node(instPort.deviceId());
    }

    private PortNumber portNumberByNodeIpAndPortName(IpAddress k8sNodeIp, String portName) {
        OpenstackNode osNode = osNodeByNodeIp(k8sNodeIp);

        if (osNode == null) {
            return null;
        }

        return osNode.portNumByName(portName);
    }

    private Port phyPortByInstPort(InstancePort instPort) {
        Network network = osNetworkService.network(instPort.networkId());

        if (network == null) {
            log.warn("The network does not exist");
            return null;
        }

        return deviceService.getPorts(instPort.deviceId()).stream()
                .filter(port -> {
                    String annotPortName = port.annotations().value(PORT_NAME);
                    String portName = structurePortName(INTEGRATION_TO_PHYSICAL_PREFIX
                            + network.getProviderPhyNet());
                    return Objects.equals(annotPortName, portName);
                }).findAny().orElse(null);
    }
}