ONOS-5182 Simplified OpenStack networking application structure
Change-Id: Ic7941f2c9a2febec4f24745278c4c305a3937097
diff --git a/apps/openstacknetworking/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingManager.java b/apps/openstacknetworking/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingManager.java
new file mode 100644
index 0000000..2482333
--- /dev/null
+++ b/apps/openstacknetworking/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingManager.java
@@ -0,0 +1,482 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.impl;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
+import org.onlab.util.Tools;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.core.GroupId;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackNetwork;
+import org.onosproject.openstackinterface.OpenstackRouter;
+import org.onosproject.openstackinterface.OpenstackRouterInterface;
+import org.onosproject.openstackinterface.OpenstackSubnet;
+import org.onosproject.openstacknetworking.api.Constants;
+import org.onosproject.openstacknetworking.api.OpenstackRoutingService;
+import org.onosproject.openstacknode.OpenstackNode;
+import org.onosproject.openstacknode.OpenstackNodeEvent;
+import org.onosproject.openstacknode.OpenstackNodeListener;
+import org.onosproject.openstacknode.OpenstackNodeService;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
+
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.api.Constants.*;
+import static org.onosproject.openstacknetworking.impl.RulePopulatorUtil.buildExtension;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.COMPUTE;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY;
+
+@Component(immediate = true)
+@Service
+public class OpenstackRoutingManager extends AbstractVmHandler implements OpenstackRoutingService {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected FlowObjectiveService flowObjectiveService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackInterfaceService openstackService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackNodeService nodeService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ScalableGatewayService gatewayService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DeviceService deviceService;
+
+ private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+ private final InternalNodeListener nodeListener = new InternalNodeListener();
+
+ private ApplicationId appId;
+
+ @Activate
+ protected void activate() {
+ super.activate();
+ appId = coreService.registerApplication(ROUTING_APP_ID);
+ nodeService.addListener(nodeListener);
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ super.deactivate();
+ nodeService.removeListener(nodeListener);
+ log.info("stopped");
+ }
+
+ @Override
+ protected void hostDetected(Host host) {
+ // Installs forwarding flow rules to VMs in different nodes and different subnets
+ // that are connected via a router
+ Optional<OpenstackRouter> routerOfTheHost = getRouter(host);
+
+ if (!routerOfTheHost.isPresent()) {
+ return;
+ }
+
+ routableSubNets(routerOfTheHost.get().id()).stream()
+ .filter(subnet -> !subnet.id().equals(host.annotations().value(SUBNET_ID)))
+ .forEach(subnet -> setForwardingRulesAmongHostsInDifferentCnodes(host, getHosts(subnet), true));
+ }
+
+ @Override
+ protected void hostRemoved(Host host) {
+ // Removes forwarding flow rules to VMs in different nodes and different subnets
+ // that are connected via a router
+ Optional<OpenstackRouter> routerOfTheHost = getRouter(host);
+ if (!routerOfTheHost.isPresent()) {
+ return;
+ }
+
+ routableSubNets(routerOfTheHost.get().id()).stream()
+ .filter(subnet -> !subnet.id().equals(host.annotations().value(SUBNET_ID)))
+ .forEach(subnet -> setForwardingRulesAmongHostsInDifferentCnodes(host, getHosts(subnet), false));
+ }
+
+ @Override
+ public void createRouter(OpenstackRouter osRouter) {
+ }
+
+ @Override
+ public void updateRouter(OpenstackRouter osRouter) {
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ routableSubNets(osRouter.id()).stream()
+ .forEach(subnet -> setExternalConnection(osRouter, subnet, true));
+
+ log.info("Connected external gateway {} to router {}",
+ osRouter.gatewayExternalInfo().externalFixedIps(),
+ osRouter.name());
+ } else {
+ routableSubNets(osRouter.id()).stream()
+ .forEach(subnet -> setExternalConnection(osRouter, subnet, false));
+
+ log.info("Disconnected external gateway from router {}",
+ osRouter.name());
+ }
+ }
+
+ @Override
+ public void removeRouter(String osRouterId) {
+ // Nothing to do
+ // All interfaces need to be removed before the router is removed,
+ // and all related flow rues are removed when the interfaces are removed.
+ }
+
+ @Override
+ public void addRouterInterface(OpenstackRouterInterface routerIfaceAdded) {
+ OpenstackRouter osRouter = openstackRouter(routerIfaceAdded.id());
+ OpenstackSubnet osSubnetAdded = openstackService.subnet(routerIfaceAdded.subnetId());
+ if (osRouter == null || osSubnetAdded == null) {
+ log.warn("Failed to add router interface {}", routerIfaceAdded);
+ return;
+ }
+ handleRouterInterfaces(osRouter, osSubnetAdded);
+ }
+
+ @Override
+ public void removeRouterInterface(OpenstackRouterInterface routerIface) {
+ OpenstackRouter osRouter = openstackService.router(routerIface.id());
+ OpenstackSubnet osSubnetRemoved = openstackService.subnet(routerIface.subnetId());
+ if (osRouter == null) {
+ log.warn("Failed to remove router interface {}", routerIface);
+ return;
+ }
+ handleRouterInterfacesRemoved(osRouter, osSubnetRemoved);
+
+ log.info("Disconnected {} from router {}", osSubnetRemoved.cidr(), osRouter.name());
+ }
+
+ private void handleRouterInterfaces(OpenstackRouter osRouter, OpenstackSubnet osSubnetAdded) {
+ OpenstackNetwork osNetworkAdded = openstackService.network(osSubnetAdded.networkId());
+ if (osNetworkAdded == null) { // in case of external network subnet
+ return;
+ }
+
+ // Sets flow rules for routing among subnets connected to a router.
+ setRoutesAmongSubnets(osRouter, osSubnetAdded, true);
+
+ // Sets flow rules for forwarding "packets going to external networks" to gateway nodes.
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ setExternalConnection(osRouter, osSubnetAdded, true);
+ }
+
+ // Sets flow rules to handle ping to the virtual gateway.
+ Ip4Address vGatewayIp = Ip4Address.valueOf(osSubnetAdded.gatewayIp());
+ gatewayService.getGatewayDeviceIds()
+ .forEach(deviceId -> setGatewayIcmpRule(vGatewayIp, deviceId, true));
+
+ // Sets east-west routing rules for VMs in different Cnode to Switching Table.
+ setForwardingRulesForEastWestRouting(osRouter, osSubnetAdded, true);
+
+ }
+
+ private void handleRouterInterfacesRemoved(OpenstackRouter osRouter, OpenstackSubnet osSubnetRemoved) {
+
+ // Removes flow rules for routing among subnets connected to a router.
+ setRoutesAmongSubnets(osRouter, osSubnetRemoved, false);
+
+ // Removes flow rules for forwarding "packets going to external networks" to gateway nodes.
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ setExternalConnection(osRouter, osSubnetRemoved, false);
+ }
+
+ // Removes flow rules to handle ping to the virtual gateway.
+ Ip4Address vGatewayIp = Ip4Address.valueOf(osSubnetRemoved.gatewayIp());
+ gatewayService.getGatewayDeviceIds()
+ .forEach(deviceId -> setGatewayIcmpRule(vGatewayIp, deviceId, false));
+
+ // Removes east-west routing rules for VMs in different Cnode to Switching Table.
+ setForwardingRulesForEastWestRouting(osRouter, osSubnetRemoved, false);
+
+ // Resets east-west routing rules for VMs in different Cnode to Switching Table.
+ routableSubNets(osRouter.id()).stream()
+ .forEach(subnet -> setForwardingRulesForEastWestRouting(osRouter, subnet, true));
+ }
+
+ private void setRoutesAmongSubnets(OpenstackRouter osRouter, OpenstackSubnet osSubnetAdded, boolean install) {
+ Set<OpenstackSubnet> routableSubNets = routableSubNets(osRouter.id());
+ if (routableSubNets.size() < 2) {
+ // no other subnet interface is connected to this router, do nothing
+ return;
+ }
+
+ Map<String, String> vniMap = new HashMap<>();
+ openstackService.networks().forEach(n -> vniMap.put(n.id(), n.segmentId()));
+
+ routableSubNets.stream()
+ .filter(subnet -> !subnet.id().equals(osSubnetAdded.id()))
+ .filter(subnet -> vniMap.get(subnet.networkId()) != null)
+ .forEach(subnet -> nodeService.completeNodes().stream()
+ .filter(node -> node.type().equals(COMPUTE))
+ .forEach(node -> {
+ setRoutingRules(node.intBridge(),
+ Integer.parseInt(vniMap.get(subnet.networkId())),
+ Integer.parseInt(vniMap.get(osSubnetAdded.networkId())),
+ subnet, osSubnetAdded, install);
+ setRoutingRules(node.intBridge(),
+ Integer.parseInt(vniMap.get(osSubnetAdded.networkId())),
+ Integer.parseInt(vniMap.get(subnet.networkId())),
+ osSubnetAdded, subnet, install);
+ }
+ ));
+ }
+
+ private void setRoutingRules(DeviceId deviceId, int srcVni, int dstVni,
+ OpenstackSubnet subnetSrc, OpenstackSubnet subnetDst, boolean install) {
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(srcVni)
+ .matchIPSrc(IpPrefix.valueOf(subnetSrc.cidr()))
+ .matchIPDst(IpPrefix.valueOf(subnetDst.cidr()));
+
+ tBuilder.setTunnelId(dstVni);
+
+ RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(),
+ tBuilder.build(), ForwardingObjective.Flag.SPECIFIC, EW_ROUTING_RULE_PRIORITY, install);
+
+ // Flow rules for destination is in different subnet and different node,
+ // because VNI is converted to destination VNI in the source VM node.
+ sBuilder = DefaultTrafficSelector.builder();
+ tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(dstVni)
+ .matchIPSrc(IpPrefix.valueOf(subnetSrc.cidr()))
+ .matchIPDst(IpPrefix.valueOf(subnetDst.cidr()));
+
+ tBuilder.setTunnelId(dstVni);
+
+ RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(),
+ tBuilder.build(), ForwardingObjective.Flag.SPECIFIC, EW_ROUTING_RULE_PRIORITY, install);
+ }
+
+ private void setExternalConnection(OpenstackRouter osRouter, OpenstackSubnet osSubNet, boolean install) {
+ if (!osRouter.gatewayExternalInfo().isEnablePnat()) {
+ log.debug("Source NAT is disabled");
+ return;
+ }
+
+ //OpenstackSubnet osSubNet = openstackService.subnet(osSubNetId);
+ OpenstackNetwork osNet = openstackService.network(osSubNet.networkId());
+
+ nodeService.completeNodes().stream()
+ .filter(node -> node.type().equals(COMPUTE))
+ .forEach(node -> setRulesToGateway(
+ node.intBridge(),
+ gatewayService.getGatewayGroupId(node.intBridge()),
+ Long.valueOf(osNet.segmentId()), osSubNet.cidr(), install));
+
+ // Is this for PNAT ??
+ setRulesForGatewayToController(Long.valueOf(osNet.segmentId()), osSubNet.cidr(), install);
+ }
+
+ private void setRulesToGateway(DeviceId deviceId, GroupId groupId, long vni, String cidr, boolean install) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchIPSrc(IpPrefix.valueOf(cidr))
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+
+ tBuilder.group(groupId);
+
+ RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(),
+ tBuilder.build(), ForwardingObjective.Flag.SPECIFIC, ROUTING_RULE_PRIORITY, install);
+ }
+
+ private void setRulesForGatewayToController(long vni, String subNetCidr, boolean install) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchIPSrc(IpPrefix.valueOf(subNetCidr))
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+ tBuilder.setOutput(PortNumber.CONTROLLER);
+
+ gatewayService.getGatewayDeviceIds().forEach(deviceId ->
+ RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(),
+ tBuilder.build(), ForwardingObjective.Flag.VERSATILE, ROUTING_RULE_PRIORITY, install));
+ }
+
+ private void setGatewayIcmpRule(Ip4Address gatewayIp, DeviceId deviceId, boolean install) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(IPv4.PROTOCOL_ICMP)
+ .matchIPDst(gatewayIp.toIpPrefix());
+
+ tBuilder.setOutput(PortNumber.CONTROLLER);
+
+ RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(),
+ tBuilder.build(), ForwardingObjective.Flag.VERSATILE, GATEWAY_ICMP_PRIORITY, install);
+ }
+
+ private void setForwardingRulesForEastWestRouting(OpenstackRouter router, OpenstackSubnet subnetAdded,
+ boolean install) {
+
+ Set<OpenstackSubnet> subnets = routableSubNets(router.id());
+
+ Set<Host> hosts = Tools.stream(hostService.getHosts())
+ .filter(h -> getVni(h).equals(openstackService.network(subnetAdded.networkId()).segmentId()))
+ .collect(Collectors.toSet());
+
+ subnets.stream()
+ .filter(subnet -> !subnet.id().equals(subnetAdded.id()))
+ .forEach(subnet -> getHosts(subnet)
+ .forEach(h -> setForwardingRulesAmongHostsInDifferentCnodes(h, hosts, install)));
+ }
+
+ private void setForwardingRulesAmongHostsInDifferentCnodes(Host host, Set<Host> remoteHosts, boolean install) {
+ Ip4Address localVmIp = getIp(host);
+ DeviceId localDeviceId = host.location().deviceId();
+ Optional<IpAddress> localDataIp = nodeService.dataIp(localDeviceId);
+
+ if (!localDataIp.isPresent()) {
+ log.debug("Failed to get data IP for device {}",
+ host.location().deviceId());
+ return;
+ }
+
+ remoteHosts.stream()
+ .filter(remoteHost -> !host.location().deviceId().equals(remoteHost.location().deviceId()))
+ .forEach(remoteVm -> {
+ Optional<IpAddress> remoteDataIp = nodeService.dataIp(remoteVm.location().deviceId());
+ if (remoteDataIp.isPresent()) {
+ setVxLanFlowRule(getVni(remoteVm),
+ localDeviceId,
+ remoteDataIp.get().getIp4Address(),
+ getIp(remoteVm), install);
+
+ setVxLanFlowRule(getVni(host),
+ remoteVm.location().deviceId(),
+ localDataIp.get().getIp4Address(),
+ localVmIp, install);
+ }
+ });
+ }
+
+ private void setVxLanFlowRule(String vni, DeviceId deviceId, Ip4Address remoteIp,
+ Ip4Address vmIp, boolean install) {
+ Optional<PortNumber> tunnelPort = nodeService.tunnelPort(deviceId);
+ if (!tunnelPort.isPresent()) {
+ log.warn("Failed to get tunnel port from {}", deviceId);
+ return;
+ }
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(Long.parseLong(vni))
+ .matchIPDst(vmIp.toIpPrefix());
+ tBuilder.extension(buildExtension(deviceService, deviceId, remoteIp), deviceId)
+ .setOutput(tunnelPort.get());
+
+ RulePopulatorUtil.setRule(flowObjectiveService, appId, deviceId, sBuilder.build(),
+ tBuilder.build(), ForwardingObjective.Flag.SPECIFIC, SWITCHING_RULE_PRIORITY, install);
+ }
+
+
+ private OpenstackRouter openstackRouter(String routerId) {
+ return openstackService.routers().stream().filter(r ->
+ r.id().equals(routerId)).iterator().next();
+ }
+
+ @Override
+ public void reinstallVmFlow(Host host) {
+ // TODO: implements later
+ }
+
+ @Override
+ public void purgeVmFlow(Host host) {
+ // TODO: implements later
+ }
+
+ private class InternalNodeListener implements OpenstackNodeListener {
+
+ @Override
+ public void event(OpenstackNodeEvent event) {
+ OpenstackNode node = event.node();
+
+ switch (event.type()) {
+ case COMPLETE:
+ case INCOMPLETE:
+ log.info("COMPLETE node {} detected", node.hostname());
+ eventExecutor.execute(() -> {
+ if (node.type() == GATEWAY) {
+ GatewayNode gnode = GatewayNode.builder()
+ .gatewayDeviceId(node.intBridge())
+ .dataIpAddress(node.dataIp().getIp4Address())
+ .uplinkIntf(node.externalPortName().get())
+ .build();
+ gatewayService.addGatewayNode(gnode);
+ }
+ });
+ openstackService.routers().stream()
+ .forEach(router -> routableSubNets(router.id()).stream()
+ .forEach(subnet -> handleRouterInterfaces(router, subnet)));
+ break;
+ case INIT:
+ case DEVICE_CREATED:
+ default:
+ break;
+ }
+ }
+ }
+}