apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingIcmpHandler.java

/*
 * Copyright 2016-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.onosproject.openstacknetworking.impl;

import com.google.common.base.Strings;
import org.onlab.packet.Ethernet;
import org.onlab.packet.ICMP;
import org.onlab.packet.ICMPEcho;
import org.onlab.packet.IPv4;
import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
import org.onlab.packet.VlanId;
import org.onlab.util.KryoNamespace;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.packet.DefaultOutboundPacket;
import org.onosproject.net.packet.InboundPacket;
import org.onosproject.net.packet.OutboundPacket;
import org.onosproject.net.packet.PacketContext;
import org.onosproject.net.packet.PacketProcessor;
import org.onosproject.net.packet.PacketService;
import org.onosproject.openstacknetworking.api.Constants;
import org.onosproject.openstacknetworking.api.ExternalPeerRouter;
import org.onosproject.openstacknetworking.api.InstancePort;
import org.onosproject.openstacknetworking.api.InstancePortService;
import org.onosproject.openstacknetworking.api.OpenstackFlowRuleService;
import org.onosproject.openstacknetworking.api.OpenstackNetworkService;
import org.onosproject.openstacknetworking.api.OpenstackRouterService;
import org.onosproject.openstacknode.api.OpenstackNode;
import org.onosproject.openstacknode.api.OpenstackNodeEvent;
import org.onosproject.openstacknode.api.OpenstackNodeListener;
import org.onosproject.openstacknode.api.OpenstackNodeService;
import org.onosproject.store.serializers.KryoNamespaces;
import org.onosproject.store.service.ConsistentMap;
import org.onosproject.store.service.Serializer;
import org.onosproject.store.service.StorageService;
import org.openstack4j.model.network.ExternalGateway;
import org.openstack4j.model.network.Port;
import org.openstack4j.model.network.Router;
import org.openstack4j.model.network.RouterInterface;
import org.openstack4j.model.network.Subnet;
import org.openstack4j.openstack.networking.domain.NeutronIP;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.slf4j.Logger;

import java.nio.ByteBuffer;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.packet.ICMP.TYPE_ECHO_REPLY;
import static org.onlab.packet.ICMP.TYPE_ECHO_REQUEST;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_GATEWAY_MAC;
import static org.onosproject.openstacknetworking.api.Constants.GW_COMMON_TABLE;
import static org.onosproject.openstacknetworking.api.Constants.OPENSTACK_NETWORKING_APP_ID;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_INTERNAL_ROUTING_RULE;
import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.GATEWAY;
import static org.slf4j.LoggerFactory.getLogger;

/**
 * Handles ICMP packet received from a gateway node.
 * For a request for virtual network subnet gateway, it generates fake ICMP reply.
 * For a request for the external network, it does source NAT with the public IP and
 * forward the request to the external only if the requested virtual subnet has
 * external connectivity.
 */
@Component(immediate = true)
public class OpenstackRoutingIcmpHandler {

    protected final Logger log = getLogger(getClass());

    private static final String ERR_REQ = "Failed to handle ICMP request: ";
    private static final String ERR_DUPLICATE = " already exists";

    private static final String VXLAN = "VXLAN";
    private static final String VLAN = "VLAN";

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected CoreService coreService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected PacketService packetService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected StorageService storageService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected OpenstackNodeService osNodeService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected InstancePortService instancePortService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected OpenstackNetworkService osNetworkService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected OpenstackRouterService osRouterService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected LeadershipService leadershipService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected OpenstackFlowRuleService osFlowRuleService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected ClusterService clusterService;

    private final ExecutorService eventExecutor = newSingleThreadExecutor(
            groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
    private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
    private ConsistentMap<String, InstancePort> icmpInfoMap;
    private final OpenstackNodeListener osNodeListener = new InternalNodeEventListener();

    private static final KryoNamespace SERIALIZER_ICMP_MAP = KryoNamespace.newBuilder()
            .register(KryoNamespaces.API)
            .register(InstancePort.class)
            .register(DefaultInstancePort.class)
            .register(InstancePort.State.class)
            .build();

    private ApplicationId appId;
    private NodeId localNodeId;

    @Activate
    protected void activate() {
        appId = coreService.registerApplication(OPENSTACK_NETWORKING_APP_ID);
        packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
        localNodeId = clusterService.getLocalNode().id();
        leadershipService.runForLeadership(appId.name());
        osNodeService.addListener(osNodeListener);

        icmpInfoMap = storageService.<String, InstancePort>consistentMapBuilder()
                .withSerializer(Serializer.using(SERIALIZER_ICMP_MAP))
                .withName("openstack-icmpmap")
                .withApplicationId(appId)
                .build();

        log.info("Started");
    }

    @Deactivate
    protected void deactivate() {
        packetService.removeProcessor(packetProcessor);
        eventExecutor.shutdown();
        leadershipService.withdraw(appId.name());
        osNodeService.removeListener(osNodeListener);

        log.info("Stopped");
    }

    private class InternalNodeEventListener implements OpenstackNodeListener {
        @Override
        public boolean isRelevant(OpenstackNodeEvent event) {
            // do not allow to proceed without leadership
            NodeId leader = leadershipService.getLeader(appId.name());
            return Objects.equals(localNodeId, leader) && event.subject().type() == GATEWAY;
        }

        @Override
        public void event(OpenstackNodeEvent event) {
            OpenstackNode osNode = event.subject();
            switch (event.type()) {
                case OPENSTACK_NODE_COMPLETE:
                    eventExecutor.execute(() -> setIcmpReplyRules(osNode.intgBridge(), true));
                    break;
                case OPENSTACK_NODE_INCOMPLETE:
                    eventExecutor.execute(() -> setIcmpReplyRules(osNode.intgBridge(), false));
                    break;
                default:
                    break;
            }
        }

        private void setIcmpReplyRules(DeviceId deviceId, boolean install) {
            // Sends ICMP response to controller for SNATing ingress traffic
            TrafficSelector selector = DefaultTrafficSelector.builder()
                    .matchEthType(Ethernet.TYPE_IPV4)
                    .matchIPProtocol(IPv4.PROTOCOL_ICMP)
                    .matchIcmpType(ICMP.TYPE_ECHO_REPLY)
                    .build();

            TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                    .punt()
                    .build();

            osFlowRuleService.setRule(
                    appId,
                    deviceId,
                    selector,
                    treatment,
                    PRIORITY_INTERNAL_ROUTING_RULE,
                    GW_COMMON_TABLE,
                    install);
        }
    }

    private class InternalPacketProcessor implements PacketProcessor {

        @Override
        public void process(PacketContext context) {
            if (context.isHandled()) {
                return;
            }

            eventExecutor.execute(() -> {
                Set<DeviceId> gateways = osNodeService.completeNodes(GATEWAY)
                        .stream().map(OpenstackNode::intgBridge)
                        .collect(Collectors.toSet());

                if (!gateways.isEmpty() &&
                        !gateways.contains(context.inPacket().receivedFrom().deviceId())) {
                    return;
                }

                InboundPacket pkt = context.inPacket();
                Ethernet ethernet = pkt.parsed();
                if (ethernet == null || ethernet.getEtherType() != Ethernet.TYPE_IPV4) {
                    return;
                }

                IPv4 iPacket = (IPv4) ethernet.getPayload();

                if (iPacket.getProtocol() == IPv4.PROTOCOL_ICMP) {
                    processIcmpPacket(context, ethernet);
                }
            });
        }

        private void processIcmpPacket(PacketContext context, Ethernet ethernet) {
            IPv4 ipPacket = (IPv4) ethernet.getPayload();
            ICMP icmp = (ICMP) ipPacket.getPayload();
            log.trace("Processing ICMP packet source MAC:{}, source IP:{}," +
                            "dest MAC:{}, dest IP:{}",
                    ethernet.getSourceMAC(),
                    IpAddress.valueOf(ipPacket.getSourceAddress()),
                    ethernet.getDestinationMAC(),
                    IpAddress.valueOf(ipPacket.getDestinationAddress()));

            switch (icmp.getIcmpType()) {
                case TYPE_ECHO_REQUEST:
                    if (handleEchoRequest(context.inPacket().receivedFrom().deviceId(),
                            ethernet.getSourceMAC(),
                            ipPacket,
                            icmp)) {
                        context.block();
                    }
                    break;
                case TYPE_ECHO_REPLY:
                    if (handleEchoReply(ipPacket, icmp)) {
                        context.block();
                    }
                    break;
                default:
                    break;
            }
        }

        private boolean handleEchoRequest(DeviceId srcDevice, MacAddress srcMac, IPv4 ipPacket,
                                          ICMP icmp) {
            //We only handles a request from an instance port
            //In case of ehco request to SNAT ip address from an external router, we intentionally ignore it
            InstancePort instPort = instancePortService.instancePort(srcMac);
            if (instPort == null) {
                log.warn(ERR_REQ + "unknown source host(MAC:{})", srcMac);
                return false;
            }

            IpAddress srcIp = IpAddress.valueOf(ipPacket.getSourceAddress());
            IpAddress dstIp = IpAddress.valueOf(ipPacket.getDestinationAddress());

            Subnet srcSubnet = getSourceSubnet(instPort);
            if (srcSubnet == null) {
                log.warn(ERR_REQ + "unknown source subnet(IP:{})", srcIp);
                return false;
            }

            if (Strings.isNullOrEmpty(srcSubnet.getGateway())) {
                log.warn(ERR_REQ + "source subnet(ID:{}, CIDR:{}) has no gateway",
                        srcSubnet.getId(), srcSubnet.getCidr());
                return false;
            }

            if (isForSubnetGateway(IpAddress.valueOf(ipPacket.getDestinationAddress()),
                    srcSubnet)) {
                // this is a request to a subnet gateway
                log.trace("Icmp request to gateway {} from {}", dstIp, srcIp);
                processRequestForGateway(ipPacket, instPort);
            } else {
                // this is a request to an external network
                log.trace("Icmp request to external {} from {}", dstIp, srcIp);

                RouterInterface routerInterface = routerInterface(srcSubnet);
                if (routerInterface == null) {
                    log.warn(ERR_REQ + "failed to get router interface");
                    return false;
                }

                ExternalGateway externalGateway = externalGateway(routerInterface);
                if (externalGateway == null) {
                    log.warn(ERR_REQ + "failed to get external gateway");
                    return false;
                }

                ExternalPeerRouter externalPeerRouter = osNetworkService.externalPeerRouter(externalGateway);
                if (externalPeerRouter == null) {
                    log.warn(ERR_REQ + "failed to get external peer router");
                    return false;
                }

                IpAddress externalIp = getExternalIp(externalGateway, routerInterface);
                if (externalIp == null) {
                    log.warn(ERR_REQ + "failed to get external ip");
                    return false;
                }

                String icmpInfoKey = icmpInfoKey(icmp,
                        externalIp.toString(),
                        IPv4.fromIPv4Address(ipPacket.getDestinationAddress()));
                log.trace("Created icmpInfo key is {}", icmpInfoKey);

                sendRequestForExternal(ipPacket, srcDevice, externalIp, externalPeerRouter);

                try {
                    icmpInfoMap.compute(icmpInfoKey, (id, existing) -> {
                        checkArgument(existing == null, ERR_DUPLICATE);
                        return instPort;
                    });
                } catch (IllegalArgumentException e) {
                    log.warn("IllegalArgumentException occurred because of {}", e.toString());
                    return false;
                }
            }
            return true;
        }

        private String icmpInfoKey(ICMP icmp, String srcIp, String dstIp) {
            return String.valueOf(getIcmpId(icmp))
                    .concat(srcIp)
                    .concat(dstIp);
        }
        private RouterInterface routerInterface(Subnet subnet) {
            checkNotNull(subnet);
            return osRouterService.routerInterfaces().stream()
                    .filter(i -> Objects.equals(i.getSubnetId(), subnet.getId()))
                    .findAny().orElse(null);
        }

        private ExternalGateway externalGateway(RouterInterface osRouterIface) {
            checkNotNull(osRouterIface);
            Router osRouter = osRouterService.router(osRouterIface.getId());
            if (osRouter == null) {
                return null;
            }
            if (osRouter.getExternalGatewayInfo() == null) {
                return null;
            }
            return osRouter.getExternalGatewayInfo();
        }

        private boolean handleEchoReply(IPv4 ipPacket, ICMP icmp) {
            String icmpInfoKey = icmpInfoKey(icmp,
                    IPv4.fromIPv4Address(ipPacket.getDestinationAddress()),
                    IPv4.fromIPv4Address(ipPacket.getSourceAddress()));
            log.trace("Retrieved icmpInfo key is {}", icmpInfoKey);

            if (icmpInfoMap.get(icmpInfoKey) != null) {
                processReplyFromExternal(ipPacket, icmpInfoMap.get(icmpInfoKey).value());
                icmpInfoMap.remove(icmpInfoKey);
                return true;
            } else {
                log.debug("No ICMP Info for ICMP packet");
                return false;
            }
        }

        private Subnet getSourceSubnet(InstancePort instance) {
            checkNotNull(instance);

            Port osPort = osNetworkService.port(instance.portId());
            return osNetworkService.subnets(osPort.getNetworkId())
                    .stream().findAny().orElse(null);
        }

        private boolean isForSubnetGateway(IpAddress dstIp, Subnet srcSubnet) {
            RouterInterface osRouterIface = osRouterService.routerInterfaces().stream()
                    .filter(i -> Objects.equals(i.getSubnetId(), srcSubnet.getId()))
                    .findAny().orElse(null);
            if (osRouterIface == null) {
                log.trace(ERR_REQ + "source subnet(ID:{}, CIDR:{}) has no router",
                        srcSubnet.getId(), srcSubnet.getCidr());
                return false;
            }

            Router osRouter = osRouterService.router(osRouterIface.getId());
            Set<IpAddress> routableGateways = osRouterService.routerInterfaces(osRouter.getId())
                    .stream()
                    .map(iface -> osNetworkService.subnet(iface.getSubnetId()).getGateway())
                    .map(IpAddress::valueOf)
                    .collect(Collectors.toSet());

            return routableGateways.contains(dstIp);
        }

        private IpAddress getExternalIp(ExternalGateway externalGateway, RouterInterface osRouterIface) {
            checkNotNull(externalGateway);
            checkNotNull(osRouterIface);

            Router osRouter = osRouterService.router(osRouterIface.getId());
            if (osRouter == null) {
                return null;
            }

            Port exGatewayPort = osNetworkService.ports(externalGateway.getNetworkId())
                    .stream()
                    .filter(port -> Objects.equals(port.getDeviceId(), osRouter.getId()))
                    .findAny().orElse(null);
            if (exGatewayPort == null) {
                final String error = String.format(ERR_REQ +
                                "no external gateway port for router (ID:%s, name:%s)",
                        osRouter.getId(), osRouter.getName());
                throw new IllegalStateException(error);
            }
            Optional<NeutronIP> externalIpAddress =
                    (Optional<NeutronIP>) exGatewayPort.getFixedIps().stream().findFirst();
            if (!externalIpAddress.isPresent() || externalIpAddress.get().getIpAddress() == null) {
                final String error = String.format(ERR_REQ +
                                "no external gateway IP address for router (ID:%s, name:%s)",
                        osRouter.getId(), osRouter.getName());
                log.warn(error);
                return null;
            }

            return IpAddress.valueOf(externalIpAddress.get().getIpAddress());
        }

        private void processRequestForGateway(IPv4 ipPacket, InstancePort instPort) {
            ICMP icmpReq = (ICMP) ipPacket.getPayload();
            icmpReq.setChecksum((short) 0);
            icmpReq.setIcmpType(TYPE_ECHO_REPLY);

            int destinationAddress = ipPacket.getSourceAddress();

            ipPacket.setSourceAddress(ipPacket.getDestinationAddress())
                    .setDestinationAddress(destinationAddress)
                    .resetChecksum();

            ipPacket.setPayload(icmpReq);
            Ethernet icmpReply = new Ethernet();
            icmpReply.setEtherType(Ethernet.TYPE_IPV4)
                    .setSourceMACAddress(Constants.DEFAULT_GATEWAY_MAC)
                    .setDestinationMACAddress(instPort.macAddress())
                    .setPayload(ipPacket);

            sendReply(icmpReply, instPort);
        }

        private void sendRequestForExternal(IPv4 ipPacket, DeviceId srcDevice,
                                            IpAddress srcNatIp, ExternalPeerRouter externalPeerRouter) {
            ICMP icmpReq = (ICMP) ipPacket.getPayload();
            icmpReq.resetChecksum();
            ipPacket.setSourceAddress(srcNatIp.getIp4Address().toInt()).resetChecksum();
            ipPacket.setPayload(icmpReq);

            Ethernet icmpRequestEth = new Ethernet();
            icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
                    .setSourceMACAddress(DEFAULT_GATEWAY_MAC)
                    .setDestinationMACAddress(externalPeerRouter.macAddress());

            if (!externalPeerRouter.vlanId().equals(VlanId.NONE)) {
                icmpRequestEth.setVlanID(externalPeerRouter.vlanId().toShort());
            }

            icmpRequestEth.setPayload(ipPacket);

            OpenstackNode osNode = osNodeService.node(srcDevice);
            if (osNode == null) {
                final String error = String.format("Cannot find openstack node for %s",
                        srcDevice);
                throw new IllegalStateException(error);
            }
            TrafficTreatment treatment = DefaultTrafficTreatment.builder()
                    .setOutput(osNode.uplinkPortNum())
                    .build();

            OutboundPacket packet = new DefaultOutboundPacket(
                    srcDevice,
                    treatment,
                    ByteBuffer.wrap(icmpRequestEth.serialize()));

            packetService.emit(packet);
        }

        private void processReplyFromExternal(IPv4 ipPacket, InstancePort instPort) {

            if (instPort.networkId() == null) {
                return;
            }

            ICMP icmpReply = (ICMP) ipPacket.getPayload();

            icmpReply.resetChecksum();

            ipPacket.setDestinationAddress(instPort.ipAddress().getIp4Address().toInt())
                    .resetChecksum();
            ipPacket.setPayload(icmpReply);

            Ethernet icmpResponseEth = new Ethernet();
            icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
                    .setSourceMACAddress(Constants.DEFAULT_GATEWAY_MAC)
                    .setDestinationMACAddress(instPort.macAddress())
                    .setPayload(ipPacket);

            sendReply(icmpResponseEth, instPort);
        }

        private void sendReply(Ethernet icmpReply, InstancePort instPort) {
            TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
                    .setOutput(instPort.portNumber());

            String netId = instPort.networkId();
            String segId = osNetworkService.segmentId(netId);

            switch (osNetworkService.networkType(netId)) {
                case VXLAN:
                    tBuilder.setTunnelId(Long.valueOf(segId));
                    break;
                case VLAN:
                    tBuilder.setVlanId(VlanId.vlanId(segId));
                    break;
                default:
                    break;
            }

            OutboundPacket packet = new DefaultOutboundPacket(
                    instPort.deviceId(),
                    tBuilder.build(),
                    ByteBuffer.wrap(icmpReply.serialize()));

            packetService.emit(packet);
        }

        private short getIcmpId(ICMP icmp) {
            return ((ICMPEcho) icmp.getPayload()).getIdentifier();
        }
    }
}