| /* |
| * Copyright 2016-present Open Networking Laboratory |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.onosproject.openstacknetworking.routing; |
| |
| import org.apache.felix.scr.annotations.Activate; |
| import org.apache.felix.scr.annotations.Component; |
| import org.apache.felix.scr.annotations.Deactivate; |
| import org.apache.felix.scr.annotations.Reference; |
| import org.apache.felix.scr.annotations.ReferenceCardinality; |
| import org.apache.felix.scr.annotations.Service; |
| import org.onlab.packet.Ethernet; |
| import org.onlab.packet.IPv4; |
| import org.onlab.packet.Ip4Address; |
| import org.onlab.packet.IpAddress; |
| import org.onlab.packet.IpPrefix; |
| import org.onlab.util.Tools; |
| import org.onosproject.core.ApplicationId; |
| import org.onosproject.core.CoreService; |
| import org.onosproject.core.GroupId; |
| import org.onosproject.net.DeviceId; |
| import org.onosproject.net.Host; |
| import org.onosproject.net.PortNumber; |
| import org.onosproject.net.device.DeviceService; |
| import org.onosproject.net.flow.DefaultTrafficSelector; |
| import org.onosproject.net.flow.DefaultTrafficTreatment; |
| import org.onosproject.net.flow.TrafficSelector; |
| import org.onosproject.net.flow.TrafficTreatment; |
| import org.onosproject.net.flowobjective.FlowObjectiveService; |
| import org.onosproject.net.flowobjective.ForwardingObjective; |
| import org.onosproject.openstackinterface.OpenstackInterfaceService; |
| import org.onosproject.openstackinterface.OpenstackNetwork; |
| import org.onosproject.openstackinterface.OpenstackRouter; |
| import org.onosproject.openstackinterface.OpenstackRouterInterface; |
| import org.onosproject.openstackinterface.OpenstackSubnet; |
| import org.onosproject.openstacknetworking.AbstractVmHandler; |
| import org.onosproject.openstacknetworking.Constants; |
| import org.onosproject.openstacknetworking.OpenstackRoutingService; |
| import org.onosproject.openstacknetworking.RulePopulatorUtil; |
| import org.onosproject.openstacknode.OpenstackNode; |
| import org.onosproject.openstacknode.OpenstackNodeEvent; |
| import org.onosproject.openstacknode.OpenstackNodeListener; |
| import org.onosproject.openstacknode.OpenstackNodeService; |
| import org.onosproject.scalablegateway.api.GatewayNode; |
| import org.onosproject.scalablegateway.api.ScalableGatewayService; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import java.util.HashMap; |
| import java.util.Map; |
| import java.util.Optional; |
| import java.util.Set; |
| import java.util.concurrent.ExecutorService; |
| import java.util.stream.Collectors; |
| |
| import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; |
| import static org.onlab.util.Tools.groupedThreads; |
| import static org.onosproject.openstacknetworking.Constants.*; |
| import static org.onosproject.openstacknetworking.RulePopulatorUtil.buildExtension; |
| import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.COMPUTE; |
| import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY; |
| |
| @Component(immediate = true) |
| @Service |
| public class OpenstackRoutingManager extends AbstractVmHandler implements OpenstackRoutingService { |
| |
| private final Logger log = LoggerFactory.getLogger(getClass()); |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected FlowObjectiveService flowObjectiveService; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected OpenstackInterfaceService openstackService; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected OpenstackNodeService nodeService; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected ScalableGatewayService gatewayService; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected CoreService coreService; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected DeviceService deviceService; |
| |
| private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor( |
| groupedThreads(this.getClass().getSimpleName(), "event-handler", log)); |
| private final InternalNodeListener nodeListener = new InternalNodeListener(); |
| |
| private ApplicationId appId; |
| |
| @Activate |
| protected void activate() { |
| super.activate(); |
| appId = coreService.registerApplication(ROUTING_APP_ID); |
| nodeService.addListener(nodeListener); |
| } |
| |
| @Deactivate |
| protected void deactivate() { |
| super.deactivate(); |
| nodeService.removeListener(nodeListener); |
| log.info("stopped"); |
| } |
| |
| @Override |
| protected void hostDetected(Host host) { |
| // Installs forwarding flow rules to VMs in different nodes and different subnets |
| // that are connected via a router |
| Optional<OpenstackRouter> routerOfTheHost = getRouter(host); |
| |
| if (!routerOfTheHost.isPresent()) { |
| return; |
| } |
| |
| routableSubNets(routerOfTheHost.get().id()).stream() |
| .filter(subnet -> !subnet.id().equals(host.annotations().value(SUBNET_ID))) |
| .forEach(subnet -> setForwardingRulesAmongHostsInDifferentCnodes(host, getHosts(subnet), true)); |
| } |
| |
| @Override |
| protected void hostRemoved(Host host) { |
| // Removes forwarding flow rules to VMs in different nodes and different subnets |
| // that are connected via a router |
| Optional<OpenstackRouter> routerOfTheHost = getRouter(host); |
| if (!routerOfTheHost.isPresent()) { |
| return; |
| } |
| |
| routableSubNets(routerOfTheHost.get().id()).stream() |
| .filter(subnet -> !subnet.id().equals(host.annotations().value(SUBNET_ID))) |
| .forEach(subnet -> setForwardingRulesAmongHostsInDifferentCnodes(host, getHosts(subnet), false)); |
| } |
| |
| @Override |
| public void createRouter(OpenstackRouter osRouter) { |
| } |
| |
| @Override |
| public void updateRouter(OpenstackRouter osRouter) { |
| if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) { |
| routableSubNets(osRouter.id()).stream() |
| .forEach(subnet -> setExternalConnection(osRouter, subnet, true)); |
| |
| log.info("Connected external gateway {} to router {}", |
| osRouter.gatewayExternalInfo().externalFixedIps(), |
| osRouter.name()); |
| } else { |
| routableSubNets(osRouter.id()).stream() |
| .forEach(subnet -> setExternalConnection(osRouter, subnet, false)); |
| |
| log.info("Disconnected external gateway from router {}", |
| osRouter.name()); |
| } |
| } |
| |
| @Override |
| public void removeRouter(String osRouterId) { |
| // Nothing to do |
| // All interfaces need to be removed before the router is removed, |
| // and all related flow rues are removed when the interfaces are removed. |
| } |
| |
| @Override |
| public void addRouterInterface(OpenstackRouterInterface routerIfaceAdded) { |
| OpenstackRouter osRouter = openstackRouter(routerIfaceAdded.id()); |
| OpenstackSubnet osSubnetAdded = openstackService.subnet(routerIfaceAdded.subnetId()); |
| if (osRouter == null || osSubnetAdded == null) { |
| log.warn("Failed to add router interface {}", routerIfaceAdded); |
| return; |
| } |
| handleRouterInterfaces(osRouter, osSubnetAdded); |
| } |
| |
| @Override |
| public void removeRouterInterface(OpenstackRouterInterface routerIface) { |
| OpenstackRouter osRouter = openstackService.router(routerIface.id()); |
| OpenstackSubnet osSubnetRemoved = openstackService.subnet(routerIface.subnetId()); |
| if (osRouter == null) { |
| log.warn("Failed to remove router interface {}", routerIface); |
| return; |
| } |
| handleRouterInterfacesRemoved(osRouter, osSubnetRemoved); |
| |
| log.info("Disconnected {} from router {}", osSubnetRemoved.cidr(), osRouter.name()); |
| } |
| |
| private void handleRouterInterfaces(OpenstackRouter osRouter, OpenstackSubnet osSubnetAdded) { |
| OpenstackNetwork osNetworkAdded = openstackService.network(osSubnetAdded.networkId()); |
| if (osNetworkAdded == null) { // in case of external network subnet |
| return; |
| } |
| |
| // Sets flow rules for routing among subnets connected to a router. |
| setRoutesAmongSubnets(osRouter, osSubnetAdded, true); |
| |
| // Sets flow rules for forwarding "packets going to external networks" to gateway nodes. |
| if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) { |
| setExternalConnection(osRouter, osSubnetAdded, true); |
| } |
| |
| // Sets flow rules to handle ping to the virtual gateway. |
| Ip4Address vGatewayIp = Ip4Address.valueOf(osSubnetAdded.gatewayIp()); |
| gatewayService.getGatewayDeviceIds() |
| .forEach(deviceId -> setGatewayIcmpRule(vGatewayIp, deviceId, true)); |
| |
| // Sets east-west routing rules for VMs in different Cnode to Switching Table. |
| setForwardingRulesForEastWestRouting(osRouter, osSubnetAdded, true); |
| |
| } |
| |
| private void handleRouterInterfacesRemoved(OpenstackRouter osRouter, OpenstackSubnet osSubnetRemoved) { |
| |
| // Removes flow rules for routing among subnets connected to a router. |
| setRoutesAmongSubnets(osRouter, osSubnetRemoved, false); |
| |
| // Removes flow rules for forwarding "packets going to external networks" to gateway nodes. |
| if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) { |
| setExternalConnection(osRouter, osSubnetRemoved, false); |
| } |
| |
| // Removes flow rules to handle ping to the virtual gateway. |
| Ip4Address vGatewayIp = Ip4Address.valueOf(osSubnetRemoved.gatewayIp()); |
| gatewayService.getGatewayDeviceIds() |
| .forEach(deviceId -> setGatewayIcmpRule(vGatewayIp, deviceId, false)); |
| |
| // Removes east-west routing rules for VMs in different Cnode to Switching Table. |
| setForwardingRulesForEastWestRouting(osRouter, osSubnetRemoved, false); |
| |
| // Resets east-west routing rules for VMs in different Cnode to Switching Table. |
| routableSubNets(osRouter.id()).stream() |
| .forEach(subnet -> setForwardingRulesForEastWestRouting(osRouter, subnet, true)); |
| } |
| |
| private void setRoutesAmongSubnets(OpenstackRouter osRouter, OpenstackSubnet osSubnetAdded, boolean install) { |
| Set<OpenstackSubnet> routableSubNets = routableSubNets(osRouter.id()); |
| if (routableSubNets.size() < 2) { |
| // no other subnet interface is connected to this router, do nothing |
| return; |
| } |
| |
| Map<String, String> vniMap = new HashMap<>(); |
| openstackService.networks().forEach(n -> vniMap.put(n.id(), n.segmentId())); |
| |
| routableSubNets.stream() |
| .filter(subnet -> !subnet.id().equals(osSubnetAdded.id())) |
| .filter(subnet -> vniMap.get(subnet.networkId()) != null) |
| .forEach(subnet -> nodeService.completeNodes().stream() |
| .filter(node -> node.type().equals(COMPUTE)) |
| .forEach(node -> { |
| setRoutingRules(node.intBridge(), |
| Integer.parseInt(vniMap.get(subnet.networkId())), |
| Integer.parseInt(vniMap.get(osSubnetAdded.networkId())), |
| subnet, osSubnetAdded, install); |
| setRoutingRules(node.intBridge(), |
| Integer.parseInt(vniMap.get(osSubnetAdded.networkId())), |
| Integer.parseInt(vniMap.get(subnet.networkId())), |
| osSubnetAdded, subnet, install); |
| } |
| )); |
| } |
| |
| private void setRoutingRules(DeviceId deviceId, int srcVni, int dstVni, |
| OpenstackSubnet subnetSrc, OpenstackSubnet subnetDst, boolean install) { |
| |
| TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); |
| TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); |
| |
| sBuilder.matchEthType(Ethernet.TYPE_IPV4) |
| .matchTunnelId(srcVni) |
| .matchIPSrc(IpPrefix.valueOf(subnetSrc.cidr())) |
| .matchIPDst(IpPrefix.valueOf(subnetDst.cidr())); |
| |
| tBuilder.setTunnelId(dstVni); |
| |
| RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(), |
| tBuilder.build(), ForwardingObjective.Flag.SPECIFIC, EW_ROUTING_RULE_PRIORITY, install); |
| |
| // Flow rules for destination is in different subnet and different node, |
| // because VNI is converted to destination VNI in the source VM node. |
| sBuilder = DefaultTrafficSelector.builder(); |
| tBuilder = DefaultTrafficTreatment.builder(); |
| |
| sBuilder.matchEthType(Ethernet.TYPE_IPV4) |
| .matchTunnelId(dstVni) |
| .matchIPSrc(IpPrefix.valueOf(subnetSrc.cidr())) |
| .matchIPDst(IpPrefix.valueOf(subnetDst.cidr())); |
| |
| tBuilder.setTunnelId(dstVni); |
| |
| RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(), |
| tBuilder.build(), ForwardingObjective.Flag.SPECIFIC, EW_ROUTING_RULE_PRIORITY, install); |
| } |
| |
| private void setExternalConnection(OpenstackRouter osRouter, OpenstackSubnet osSubNet, boolean install) { |
| if (!osRouter.gatewayExternalInfo().isEnablePnat()) { |
| log.debug("Source NAT is disabled"); |
| return; |
| } |
| |
| //OpenstackSubnet osSubNet = openstackService.subnet(osSubNetId); |
| OpenstackNetwork osNet = openstackService.network(osSubNet.networkId()); |
| |
| nodeService.completeNodes().stream() |
| .filter(node -> node.type().equals(COMPUTE)) |
| .forEach(node -> setRulesToGateway( |
| node.intBridge(), |
| gatewayService.getGatewayGroupId(node.intBridge()), |
| Long.valueOf(osNet.segmentId()), osSubNet.cidr(), install)); |
| |
| // Is this for PNAT ?? |
| setRulesForGatewayToController(Long.valueOf(osNet.segmentId()), osSubNet.cidr(), install); |
| } |
| |
| private void setRulesToGateway(DeviceId deviceId, GroupId groupId, long vni, String cidr, boolean install) { |
| TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); |
| TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); |
| |
| sBuilder.matchEthType(Ethernet.TYPE_IPV4) |
| .matchTunnelId(vni) |
| .matchIPSrc(IpPrefix.valueOf(cidr)) |
| .matchEthDst(Constants.DEFAULT_GATEWAY_MAC); |
| |
| tBuilder.group(groupId); |
| |
| RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(), |
| tBuilder.build(), ForwardingObjective.Flag.SPECIFIC, ROUTING_RULE_PRIORITY, install); |
| } |
| |
| private void setRulesForGatewayToController(long vni, String subNetCidr, boolean install) { |
| TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); |
| TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); |
| |
| sBuilder.matchEthType(Ethernet.TYPE_IPV4) |
| .matchTunnelId(vni) |
| .matchIPSrc(IpPrefix.valueOf(subNetCidr)) |
| .matchEthDst(Constants.DEFAULT_GATEWAY_MAC); |
| tBuilder.setOutput(PortNumber.CONTROLLER); |
| |
| gatewayService.getGatewayDeviceIds().forEach(deviceId -> |
| RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(), |
| tBuilder.build(), ForwardingObjective.Flag.VERSATILE, ROUTING_RULE_PRIORITY, install)); |
| } |
| |
| private void setGatewayIcmpRule(Ip4Address gatewayIp, DeviceId deviceId, boolean install) { |
| TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); |
| TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); |
| |
| sBuilder.matchEthType(Ethernet.TYPE_IPV4) |
| .matchIPProtocol(IPv4.PROTOCOL_ICMP) |
| .matchIPDst(gatewayIp.toIpPrefix()); |
| |
| tBuilder.setOutput(PortNumber.CONTROLLER); |
| |
| RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(), |
| tBuilder.build(), ForwardingObjective.Flag.VERSATILE, GATEWAY_ICMP_PRIORITY, install); |
| } |
| |
| private void setForwardingRulesForEastWestRouting(OpenstackRouter router, OpenstackSubnet subnetAdded, |
| boolean install) { |
| |
| Set<OpenstackSubnet> subnets = routableSubNets(router.id()); |
| |
| Set<Host> hosts = Tools.stream(hostService.getHosts()) |
| .filter(h -> getVni(h).equals(openstackService.network(subnetAdded.networkId()).segmentId())) |
| .collect(Collectors.toSet()); |
| |
| subnets.stream() |
| .filter(subnet -> !subnet.id().equals(subnetAdded.id())) |
| .forEach(subnet -> getHosts(subnet) |
| .forEach(h -> setForwardingRulesAmongHostsInDifferentCnodes(h, hosts, install))); |
| } |
| |
| private void setForwardingRulesAmongHostsInDifferentCnodes(Host host, Set<Host> remoteHosts, boolean install) { |
| Ip4Address localVmIp = getIp(host); |
| DeviceId localDeviceId = host.location().deviceId(); |
| Optional<IpAddress> localDataIp = nodeService.dataIp(localDeviceId); |
| |
| if (!localDataIp.isPresent()) { |
| log.debug("Failed to get data IP for device {}", |
| host.location().deviceId()); |
| return; |
| } |
| |
| remoteHosts.stream() |
| .filter(remoteHost -> !host.location().deviceId().equals(remoteHost.location().deviceId())) |
| .forEach(remoteVm -> { |
| Optional<IpAddress> remoteDataIp = nodeService.dataIp(remoteVm.location().deviceId()); |
| if (remoteDataIp.isPresent()) { |
| setVxLanFlowRule(getVni(remoteVm), |
| localDeviceId, |
| remoteDataIp.get().getIp4Address(), |
| getIp(remoteVm), install); |
| |
| setVxLanFlowRule(getVni(host), |
| remoteVm.location().deviceId(), |
| localDataIp.get().getIp4Address(), |
| localVmIp, install); |
| } |
| }); |
| } |
| |
| private void setVxLanFlowRule(String vni, DeviceId deviceId, Ip4Address remoteIp, |
| Ip4Address vmIp, boolean install) { |
| Optional<PortNumber> tunnelPort = nodeService.tunnelPort(deviceId); |
| if (!tunnelPort.isPresent()) { |
| log.warn("Failed to get tunnel port from {}", deviceId); |
| return; |
| } |
| |
| TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); |
| TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); |
| |
| sBuilder.matchEthType(Ethernet.TYPE_IPV4) |
| .matchTunnelId(Long.parseLong(vni)) |
| .matchIPDst(vmIp.toIpPrefix()); |
| tBuilder.extension(buildExtension(deviceService, deviceId, remoteIp), deviceId) |
| .setOutput(tunnelPort.get()); |
| |
| RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(), |
| tBuilder.build(), ForwardingObjective.Flag.SPECIFIC, SWITCHING_RULE_PRIORITY, install); |
| } |
| |
| |
| private OpenstackRouter openstackRouter(String routerId) { |
| return openstackService.routers().stream().filter(r -> |
| r.id().equals(routerId)).iterator().next(); |
| } |
| |
| @Override |
| public void reinstallVmFlow(Host host) { |
| // TODO: implements later |
| } |
| |
| @Override |
| public void purgeVmFlow(Host host) { |
| // TODO: implements later |
| } |
| |
| private class InternalNodeListener implements OpenstackNodeListener { |
| |
| @Override |
| public void event(OpenstackNodeEvent event) { |
| OpenstackNode node = event.node(); |
| |
| switch (event.type()) { |
| case COMPLETE: |
| case INCOMPLETE: |
| log.info("COMPLETE node {} detected", node.hostname()); |
| eventExecutor.execute(() -> { |
| if (node.type() == GATEWAY) { |
| GatewayNode gnode = GatewayNode.builder() |
| .gatewayDeviceId(node.intBridge()) |
| .dataIpAddress(node.dataIp().getIp4Address()) |
| .uplinkIntf(node.externalPortName().get()) |
| .build(); |
| gatewayService.addGatewayNode(gnode); |
| } |
| }); |
| openstackService.routers().stream() |
| .forEach(router -> routableSubNets(router.id()).stream() |
| .forEach(subnet -> handleRouterInterfaces(router, subnet))); |
| break; |
| case INIT: |
| case DEVICE_CREATED: |
| default: |
| break; |
| } |
| } |
| } |
| } |