Refactored OpenstackRouting to support multiple gateway nodes
Change-Id: I6870ca9a4fd6f6b1cf2d2be72f52ef87827e1d2c
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
index 9f19a28..09b4b8a 100644
--- a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
@@ -15,8 +15,7 @@
*/
package org.onosproject.openstacknetworking.routing;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.ImmutableSet;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
@@ -24,82 +23,62 @@
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.Service;
import org.onlab.packet.Ethernet;
-import org.onlab.packet.IPv4;
import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
-import org.onlab.packet.UDP;
-import org.onlab.util.KryoNamespace;
+import org.onlab.util.Tools;
import org.onosproject.core.ApplicationId;
-import org.onosproject.core.CoreService;
-import org.onosproject.mastership.MastershipService;
-import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.core.GroupId;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Host;
-import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.host.DefaultHostDescription;
-import org.onosproject.net.host.HostDescription;
-import org.onosproject.net.host.HostEvent;
-import org.onosproject.net.host.HostListener;
-import org.onosproject.net.host.HostProvider;
-import org.onosproject.net.host.HostProviderRegistry;
-import org.onosproject.net.host.HostProviderService;
-import org.onosproject.net.host.HostService;
-import org.onosproject.net.packet.InboundPacket;
-import org.onosproject.net.packet.PacketContext;
-import org.onosproject.net.packet.PacketProcessor;
-import org.onosproject.net.packet.PacketService;
-import org.onosproject.net.provider.AbstractProvider;
-import org.onosproject.net.provider.ProviderId;
-import org.onosproject.openstackinterface.OpenstackFloatingIP;
+import org.onosproject.net.flowobjective.ForwardingObjective;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackNetwork;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstackinterface.OpenstackRouterInterface;
-import org.onosproject.openstacknetworking.OpenstackRoutingService;
-import org.onosproject.scalablegateway.api.ScalableGatewayService;
-import org.onosproject.openstacknetworking.routing.OpenstackFloatingIPHandler.Action;
+import org.onosproject.openstackinterface.OpenstackSubnet;
+import org.onosproject.openstacknetworking.AbstractVmHandler;
import org.onosproject.openstacknetworking.Constants;
+import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.onosproject.openstacknetworking.RulePopulatorUtil;
import org.onosproject.openstacknode.OpenstackNode;
import org.onosproject.openstacknode.OpenstackNodeEvent;
import org.onosproject.openstacknode.OpenstackNodeListener;
import org.onosproject.openstacknode.OpenstackNodeService;
-import org.onosproject.store.serializers.KryoNamespaces;
-import org.onosproject.store.service.ConsistentMap;
-import org.onosproject.store.service.Serializer;
-import org.onosproject.store.service.StorageService;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
-import java.util.List;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.stream.Collectors;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static org.onlab.util.Tools.groupedThreads;
-import static org.onosproject.net.AnnotationKeys.PORT_NAME;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.onosproject.openstacknetworking.RulePopulatorUtil.buildExtension;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.COMPUTE;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY;
@Component(immediate = true)
@Service
-/**
- * Populates flow rules about L3 functionality for VMs in Openstack.
- */
-public class OpenstackRoutingManager implements OpenstackRoutingService {
+public class OpenstackRoutingManager extends AbstractVmHandler implements OpenstackRoutingService {
private final Logger log = LoggerFactory.getLogger(getClass());
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected CoreService coreService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected PacketService packetService;
+ protected FlowObjectiveService flowObjectiveService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected DeviceService deviceService;
@@ -108,620 +87,392 @@
protected OpenstackInterfaceService openstackService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected FlowObjectiveService flowObjectiveService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected DriverService driverService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected StorageService storageService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected HostService hostService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected HostProviderRegistry hostProviderRegistry;
+ protected OpenstackNodeService nodeService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ScalableGatewayService gatewayService;
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected OpenstackNodeService nodeService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected MastershipService mastershipService;
+ private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+ private final InternalNodeListener nodeListener = new InternalNodeListener();
private ApplicationId appId;
- private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
- private ConsistentMap<String, OpenstackFloatingIP> floatingIpMap; // Map<FloatingIp`s Id, FloatingIp object>
- // Map<RouterInterface`s portId, Corresponded port`s network id>
- private ConsistentMap<String, String> routerInterfaceMap;
- private static final ProviderId PID = new ProviderId("of", "org.onosproject.openstackroutering", true);
- private static final String APP_ID = "org.onosproject.openstackrouting";
- private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
- private static final String FLOATING_IP_MAP_NAME = "openstackrouting-floatingip";
- private static final String TP_PORT_MAP_NAME = "openstackrouting-tpportnum";
- private static final String ROUTER_INTERFACE_MAP_NAME = "openstackrouting-routerinterface";
- private static final String COLON = ":";
- private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
- private static final int TP_PORT_MINIMUM_NUM = 1024;
- private static final int TP_PORT_MAXIMUM_NUM = 65535;
-
- private static final KryoNamespace.Builder FLOATING_IP_SERIALIZER = KryoNamespace.newBuilder()
- .register(KryoNamespaces.API)
- .register(OpenstackFloatingIP.FloatingIpStatus.class)
- .register(OpenstackFloatingIP.class);
-
- private static final KryoNamespace.Builder NUMBER_SERIALIZER = KryoNamespace.newBuilder()
- .register(KryoNamespaces.API);
-
- private static final KryoNamespace.Builder ROUTER_INTERFACE_SERIALIZER = KryoNamespace.newBuilder()
- .register(KryoNamespaces.API);
-
- private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
- private InternalHostListener internalHostListener = new InternalHostListener();
- private InternalOpenstackNodeListener internalNodeListener = new InternalOpenstackNodeListener();
- private ExecutorService l3EventExecutorService =
- Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "L3-event"));
- private ExecutorService icmpEventExecutorService =
- Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "icmp-event"));
- private ExecutorService arpEventExecutorService =
- Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "arp-event"));
- private OpenstackIcmpHandler openstackIcmpHandler;
- private OpenstackRoutingArpHandler openstackArpHandler;
- private OpenstackRoutingRulePopulator rulePopulator;
-
- private HostProviderService hostProviderService;
- private final HostProvider hostProvider = new InternalHostProvider();
@Activate
protected void activate() {
- appId = coreService.registerApplication(APP_ID);
- hostService.addListener(internalHostListener);
- nodeService.addListener(internalNodeListener);
- hostProviderService = hostProviderRegistry.register(hostProvider);
-
- floatingIpMap = storageService.<String, OpenstackFloatingIP>consistentMapBuilder()
- .withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
- .withName(FLOATING_IP_MAP_NAME)
- .withApplicationId(appId)
- .build();
- tpPortNumMap = storageService.<Integer, String>consistentMapBuilder()
- .withSerializer(Serializer.using(NUMBER_SERIALIZER.build()))
- .withName(TP_PORT_MAP_NAME)
- .withApplicationId(appId)
- .build();
- routerInterfaceMap = storageService.<String, String>consistentMapBuilder()
- .withSerializer(Serializer.using(ROUTER_INTERFACE_SERIALIZER.build()))
- .withName(ROUTER_INTERFACE_MAP_NAME)
- .withApplicationId(appId)
- .build();
-
- log.info("started");
+ super.activate();
+ appId = coreService.registerApplication(ROUTING_APP_ID);
+ nodeService.addListener(nodeListener);
}
@Deactivate
protected void deactivate() {
- packetService.removeProcessor(internalPacketProcessor);
- hostService.removeListener(internalHostListener);
- nodeService.removeListener(internalNodeListener);
-
- l3EventExecutorService.shutdown();
- icmpEventExecutorService.shutdown();
- arpEventExecutorService.shutdown();
-
- floatingIpMap.clear();
- tpPortNumMap.clear();
- routerInterfaceMap.clear();
-
+ nodeService.removeListener(nodeListener);
log.info("stopped");
}
-
@Override
- public void createFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
- floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
+ public void createRouter(OpenstackRouter osRouter) {
}
@Override
- public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
- if (!floatingIpMap.containsKey(openstackFloatingIp.id())) {
- log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIp.id());
+ public void updateRouter(OpenstackRouter osRouter) {
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ openstackService.ports().stream()
+ .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ osPort.deviceId().equals(osRouter.id()))
+ .forEach(osPort -> setExternalConnection(osRouter, osPort.networkId()));
+
+ log.info("Connected external gateway {} to router {}",
+ osRouter.gatewayExternalInfo().externalFixedIps(),
+ osRouter.name());
+ } else {
+ openstackService.ports().stream()
+ .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ osPort.deviceId().equals(osRouter.id()))
+ .forEach(osPort -> unsetExternalConnection(osRouter, osPort.networkId()));
+
+ log.info("Disconnected external gateway from router {}",
+ osRouter.name());
+ }
+ }
+
+ @Override
+ public void removeRouter(String osRouterId) {
+ // TODO implement this
+ }
+
+ @Override
+ public void addRouterInterface(OpenstackRouterInterface routerIface) {
+ OpenstackRouter osRouter = openstackRouter(routerIface.id());
+ OpenstackPort osPort = openstackService.port(routerIface.portId());
+ if (osRouter == null || osPort == null) {
+ log.warn("Failed to add router interface {}", routerIface);
return;
}
- if (openstackFloatingIp.portId() == null || openstackFloatingIp.portId().equals("null")) {
- OpenstackFloatingIP floatingIp = floatingIpMap.get(openstackFloatingIp.id()).value();
- // XXX When the VM has been removed, host information has been removed or not ???
- Optional<Host> host = hostService.getHostsByIp(openstackFloatingIp.fixedIpAddress().getIp4Address())
- .stream()
- .findFirst();
- if (!host.isPresent()) {
- log.warn("No Host info with the VM IP the Floating IP address {} is found",
- openstackFloatingIp.floatingIpAddress());
- return;
- }
- l3EventExecutorService.execute(
- new OpenstackFloatingIPHandler(rulePopulator, floatingIp, Action.DISSASSOCIATE, host.get()));
- floatingIpMap.replace(floatingIp.id(), openstackFloatingIp);
- registerFloatingIpToHostService(openstackFloatingIp, Action.DISSASSOCIATE);
- } else {
- floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
- l3EventExecutorService.execute(
- new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIp, Action.ASSOCIATE, null));
- registerFloatingIpToHostService(openstackFloatingIp, Action.ASSOCIATE);
+
+ setRoutes(osRouter, Optional.empty());
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ setExternalConnection(osRouter, osPort.networkId());
}
+ log.info("Connected {} to router {}", osPort.fixedIps(), osRouter.name());
}
@Override
- public void deleteFloatingIP(String id) {
- floatingIpMap.remove(id);
- }
-
- @Override
- public void createRouter(OpenstackRouter openstackRouter) {
- }
-
- @Override
- public void updateRouter(OpenstackRouter openstackRouter) {
- if (openstackRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
- checkExternalConnection(openstackRouter, getOpenstackRouterInterface(openstackRouter));
- } else {
- unsetExternalConnection();
- }
- }
-
- private void unsetExternalConnection() {
- Collection<OpenstackRouter> internalRouters = getExternalRouter(false);
- internalRouters.forEach(r ->
- getOpenstackRouterInterface(r).forEach(i -> rulePopulator.removeExternalRules(i)));
- }
-
- private Collection<OpenstackRouter> getExternalRouter(boolean externalConnection) {
- List<OpenstackRouter> routers;
- if (externalConnection) {
- routers = openstackService.routers()
- .stream()
- .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() > 0))
- .collect(Collectors.toList());
- } else {
- routers = openstackService.routers()
- .stream()
- .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() == 0))
- .collect(Collectors.toList());
- }
- return routers;
- }
-
- @Override
- public void deleteRouter(String id) {
- //TODO : In now, there`s nothing to do for deleteRouter process. It is reserved.
- }
-
- @Override
- public void updateRouterInterface(OpenstackRouterInterface routerInterface) {
- List<OpenstackRouterInterface> routerInterfaces = Lists.newArrayList();
- routerInterfaces.add(routerInterface);
- checkExternalConnection(getOpenstackRouter(routerInterface.id()), routerInterfaces);
- setL3Connection(getOpenstackRouter(routerInterface.id()), null);
- routerInterfaceMap.put(routerInterface.portId(), openstackService.port(routerInterface.portId()).networkId());
- }
-
- /**
- * Set flow rules for traffic between two different subnets when more than one subnets
- * connected to a router.
- *
- * @param openstackRouter OpenstackRouter Info
- * @param openstackPort OpenstackPort Info
- */
- private void setL3Connection(OpenstackRouter openstackRouter, OpenstackPort openstackPort) {
- Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(openstackRouter);
-
- if (interfaceList.size() < 2) {
+ public void removeRouterInterface(OpenstackRouterInterface routerIface) {
+ OpenstackRouter osRouter = openstackService.router(routerIface.id());
+ if (osRouter == null) {
+ log.warn("Failed to remove router interface {}", routerIface);
return;
}
- if (openstackPort == null) {
- interfaceList.forEach(i -> {
- OpenstackPort interfacePort = openstackService.port(i.portId());
- openstackService.ports()
- .stream()
- .filter(p -> p.networkId().equals(interfacePort.networkId())
- && !p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
- .forEach(p -> rulePopulator.populateL3Rules(p,
- getL3ConnectionList(p.networkId(), interfaceList)));
- });
- } else {
- rulePopulator.populateL3Rules(openstackPort, getL3ConnectionList(openstackPort.networkId(), interfaceList));
+ OpenstackSubnet osSubnet = openstackService.subnet(routerIface.subnetId());
+ OpenstackNetwork osNet = openstackService.network(osSubnet.networkId());
+
+ unsetRoutes(osRouter, osNet);
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ unsetExternalConnection(osRouter, osNet.id());
}
-
+ log.info("Disconnected {} from router {}", osSubnet.cidr(), osRouter.name());
}
- private List<OpenstackRouterInterface> getL3ConnectionList(String networkId,
- Collection<OpenstackRouterInterface> interfaceList) {
- List<OpenstackRouterInterface> targetList = Lists.newArrayList();
- interfaceList.forEach(i -> {
- OpenstackPort port = openstackService.port(i.portId());
- if (!port.networkId().equals(networkId)) {
- targetList.add(i);
- }
- });
- return targetList;
- }
-
- @Override
- public void removeRouterInterface(OpenstackRouterInterface routerInterface) {
- OpenstackRouter router = openstackService.router(routerInterface.id());
- Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(router);
- if (interfaceList.size() == 1) {
- List<OpenstackRouterInterface> newList = Lists.newArrayList();
- newList.add(routerInterface);
- interfaceList.forEach(i -> removeL3RulesForRouterInterface(i, router, newList));
- }
- removeL3RulesForRouterInterface(routerInterface, router, null);
- rulePopulator.removeExternalRules(routerInterface);
- routerInterfaceMap.remove(routerInterface.portId());
- }
-
- private void removeL3RulesForRouterInterface(OpenstackRouterInterface routerInterface, OpenstackRouter router,
- List<OpenstackRouterInterface> newList) {
- if (!routerInterfaceMap.containsKey(routerInterface.portId())) {
- log.warn("No router interface information found for {}", routerInterface.portId());
+ private void setExternalConnection(OpenstackRouter osRouter, String osNetId) {
+ if (!osRouter.gatewayExternalInfo().isEnablePnat()) {
+ log.debug("Source NAT is disabled");
return;
}
- openstackService.ports(routerInterfaceMap.get(routerInterface.portId()).value()).forEach(p -> {
- Ip4Address vmIp = (Ip4Address) p.fixedIps().values().toArray()[0];
- if (newList == null) {
- rulePopulator.removeL3Rules(vmIp,
- getL3ConnectionList(p.networkId(), getOpenstackRouterInterface(router)));
- } else {
- rulePopulator.removeL3Rules(vmIp, newList);
- }
- }
- );
+
+ // FIXME router interface is subnet specific, not network
+ OpenstackNetwork osNet = openstackService.network(osNetId);
+ populateExternalRules(osNet);
}
- @Override
- public String networkIdForRouterInterface(String portId) {
- return routerInterfaceMap.get(portId).value();
- }
-
- private Collection<OpenstackFloatingIP> associatedFloatingIps() {
- List<OpenstackFloatingIP> fIps = Lists.newArrayList();
- floatingIpMap.values()
- .stream()
- .filter(fIp -> fIp.value().portId() != null)
- .forEach(fIp -> fIps.add(fIp.value()));
- return fIps;
- }
-
- private void reloadInitL3Rules() {
-
- l3EventExecutorService.execute(() ->
- openstackService.ports()
- .stream()
- .forEach(p ->
- {
- if (p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)) {
- updateRouterInterface(portToRouterInterface(p));
- } else {
- Optional<Ip4Address> vmIp = p.fixedIps().values().stream().findAny();
- if (vmIp.isPresent()) {
- OpenstackFloatingIP floatingIP = getOpenstackFloatingIp(vmIp.get());
- if (floatingIP != null) {
- updateFloatingIP(floatingIP);
- }
- }
- }
- })
- );
- }
-
- private OpenstackRouterInterface portToRouterInterface(OpenstackPort p) {
- OpenstackRouterInterface.Builder osBuilder = new OpenstackRouterInterface.Builder()
- .id(checkNotNull(p.deviceId()))
- .tenantId(checkNotNull(openstackService.network(p.networkId()).tenantId()))
- .subnetId(checkNotNull(p.fixedIps().keySet().stream().findFirst().orElse(null)).toString())
- .portId(checkNotNull(p.id()));
-
- return osBuilder.build();
- }
-
- private class InternalPacketProcessor implements PacketProcessor {
-
- @Override
- public void process(PacketContext context) {
-
- DeviceId senderDeviceId = context.inPacket().receivedFrom().deviceId();
- if (!nodeService.routerBridge(senderDeviceId).isPresent()) {
- log.warn("No router bridge for {} is found.", senderDeviceId);
- return;
- }
- if (context.isHandled()) {
- return;
- } else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
- return;
- }
-
- InboundPacket pkt = context.inPacket();
- Ethernet ethernet = pkt.parsed();
-
- //TODO: Considers IPv6 later.
- if (ethernet == null) {
- return;
- } else if (ethernet.getEtherType() == Ethernet.TYPE_IPV4) {
- IPv4 iPacket = (IPv4) ethernet.getPayload();
- switch (iPacket.getProtocol()) {
- case IPv4.PROTOCOL_ICMP:
-
- icmpEventExecutorService.execute(() ->
- openstackIcmpHandler.processIcmpPacket(context, ethernet));
- break;
- case IPv4.PROTOCOL_UDP:
- // don't process DHCP
- UDP udpPacket = (UDP) iPacket.getPayload();
- if (udpPacket.getDestinationPort() == UDP.DHCP_SERVER_PORT &&
- udpPacket.getSourcePort() == UDP.DHCP_CLIENT_PORT) {
- break;
- }
- default:
- int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
- DeviceId deviceId = pkt.receivedFrom().deviceId();
- Port port = null;
- port = deviceService.getPort(deviceId,
- gatewayService.getUplinkPort(deviceId));
- if (port != null) {
- OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
- Ip4Address.valueOf(iPacket.getSourceAddress()));
- l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
- portNum, openstackPort, port));
-
- } else {
- log.warn("There`s no external interface");
- }
-
- break;
- }
- } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
- arpEventExecutorService.execute(() ->
- openstackArpHandler.processArpPacketFromRouter(context, ethernet));
- }
+ private void unsetExternalConnection(OpenstackRouter osRouter, String osNetId) {
+ if (!osRouter.gatewayExternalInfo().isEnablePnat()) {
+ log.debug("Source NAT is disabled");
+ return;
}
- private int getPortNum(MacAddress sourceMac, int destinationAddress) {
- int portNum = findUnusedPortNum();
- if (portNum == 0) {
- clearPortNumMap();
- portNum = findUnusedPortNum();
- }
- tpPortNumMap.put(portNum, sourceMac.toString().concat(COLON).concat(String.valueOf(destinationAddress)));
- return portNum;
- }
-
- private int findUnusedPortNum() {
- for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
- if (!tpPortNumMap.containsKey(i)) {
- return i;
- }
- }
- return 0;
- }
-
+ // FIXME router interface is subnet specific, not network
+ OpenstackNetwork osNet = openstackService.network(osNetId);
+ removeExternalRules(osNet);
}
- private boolean checkGatewayNode(DeviceId deviceId) {
- return gatewayService.getGatewayDeviceIds().contains(deviceId);
+ private void setRoutes(OpenstackRouter osRouter, Optional<Host> host) {
+ Set<OpenstackNetwork> routableNets = routableNetworks(osRouter.id());
+ if (routableNets.size() < 2) {
+ // no other subnet interface is connected to this router, do nothing
+ return;
+ }
+
+ // FIXME router interface is subnet specific, not network
+ Set<String> routableNetIds = routableNets.stream()
+ .map(OpenstackNetwork::id)
+ .collect(Collectors.toSet());
+
+ Set<Host> hosts = host.isPresent() ? ImmutableSet.of(host.get()) :
+ Tools.stream(hostService.getHosts())
+ .filter(h -> routableNetIds.contains(h.annotations().value(NETWORK_ID)))
+ .collect(Collectors.toSet());
+
+ hosts.stream().forEach(h -> populateRoutingRules(h, routableNets));
}
- private void clearPortNumMap() {
- tpPortNumMap.entrySet().forEach(e -> {
- if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
- tpPortNumMap.remove(e.getKey());
- }
+ private void unsetRoutes(OpenstackRouter osRouter, OpenstackNetwork osNet) {
+ // FIXME router interface is subnet specific, not network
+ Set<OpenstackNetwork> routableNets = routableNetworks(osRouter.id());
+ Tools.stream(hostService.getHosts())
+ .filter(h -> Objects.equals(
+ h.annotations().value(NETWORK_ID), osNet.id()))
+ .forEach(h -> removeRoutingRules(h, routableNets));
+
+ routableNets.stream().forEach(n -> {
+ Tools.stream(hostService.getHosts())
+ .filter(h -> Objects.equals(
+ h.annotations().value(NETWORK_ID),
+ n.id()))
+ .forEach(h -> removeRoutingRules(h, ImmutableSet.of(osNet)));
+ log.debug("Removed between {} to {}", n.name(), osNet.name());
});
}
- private Optional<Port> getExternalPort(DeviceId deviceId, String interfaceName) {
- return deviceService.getPorts(deviceId)
- .stream()
- .filter(p -> p.annotations().value(PORT_NAME).equals(interfaceName))
- .findAny();
- }
-
- private void checkExternalConnection(OpenstackRouter router,
- Collection<OpenstackRouterInterface> interfaces) {
- checkNotNull(router, "Router can not be null");
- checkNotNull(interfaces, "routerInterfaces can not be null");
- Ip4Address externalIp = router.gatewayExternalInfo().externalFixedIps()
- .values().stream().findFirst().orElse(null);
- if ((externalIp == null) || (!router.gatewayExternalInfo().isEnablePnat())) {
- log.debug("Not satisfied to set pnat configuration");
- return;
- }
- interfaces.forEach(this::initiateL3Rule);
- }
-
- private Optional<OpenstackRouter> getRouterfromExternalIp(Ip4Address externalIp) {
- return getExternalRouter(true)
- .stream()
- .filter(r -> r.gatewayExternalInfo()
- .externalFixedIps()
- .values()
- .stream()
- .findAny()
- .get()
- .equals(externalIp))
- .findAny();
- }
-
- private void initiateL3Rule(OpenstackRouterInterface routerInterface) {
- long vni = Long.parseLong(openstackService.network(openstackService
- .port(routerInterface.portId()).networkId()).segmentId());
- rulePopulator.populateExternalRules(vni);
- }
-
- private Collection<OpenstackRouterInterface> getOpenstackRouterInterface(OpenstackRouter router) {
- List<OpenstackRouterInterface> interfaces = Lists.newArrayList();
- openstackService.ports()
- .stream()
- .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
- && p.deviceId().equals(router.id()))
- .forEach(p -> interfaces.add(portToRouterInterface(p)));
- return interfaces;
- }
-
- private OpenstackRouter getOpenstackRouter(String id) {
+ private OpenstackRouter openstackRouter(String routerId) {
return openstackService.routers().stream().filter(r ->
- r.id().equals(id)).iterator().next();
+ r.id().equals(routerId)).iterator().next();
}
- private OpenstackPort getOpenstackPort(MacAddress sourceMac, Ip4Address ip4Address) {
- OpenstackPort openstackPort = openstackService.ports().stream()
- .filter(p -> p.macAddress().equals(sourceMac)).iterator().next();
- return openstackPort.fixedIps().values().stream().filter(ip ->
- ip.equals(ip4Address)).count() > 0 ? openstackPort : null;
- }
-
- private OpenstackFloatingIP getOpenstackFloatingIp(Ip4Address vmIp) {
- Optional<OpenstackFloatingIP> floatingIp = floatingIpMap.asJavaMap().values().stream()
- .filter(f -> f.portId() != null && f.fixedIpAddress().equals(vmIp))
- .findAny();
-
- if (floatingIp.isPresent()) {
- return floatingIp.get();
- }
- log.debug("There is no floating IP information for VM IP {}", vmIp);
-
- return null;
- }
-
- private Optional<OpenstackPort> getRouterInterfacePort(String networkId) {
-
- return openstackService.ports()
- .stream()
- .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
- && p.networkId().equals(networkId))
+ private Optional<OpenstackPort> routerIfacePort(String osNetId) {
+ // FIXME router interface is subnet specific, not network
+ return openstackService.ports().stream()
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ p.networkId().equals(osNetId))
.findAny();
}
- // TODO: Remove the function and the related codes when vRouter is running on different ONOS instance.
- private void registerFloatingIpToHostService(OpenstackFloatingIP openstackFloatingIp, Action action) {
+ private Set<OpenstackNetwork> routableNetworks(String osRouterId) {
+ // FIXME router interface is subnet specific, not network
+ return openstackService.ports().stream()
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ p.deviceId().equals(osRouterId))
+ .map(p -> openstackService.network(p.networkId()))
+ .collect(Collectors.toSet());
+ }
- Optional<Host> hostOptional = hostService.getHostsByIp(openstackFloatingIp.fixedIpAddress())
- .stream()
- .findFirst();
- if (!hostOptional.isPresent()) {
- log.warn("No host with IP {} is registered and cannot add the floating IP. ",
- openstackFloatingIp.floatingIpAddress());
+ private void populateExternalRules(OpenstackNetwork osNet) {
+ populateCnodeToGateway(Long.valueOf(osNet.segmentId()));
+ populateGatewayToController(Long.valueOf(osNet.segmentId()));
+ }
+
+ private void removeExternalRules(OpenstackNetwork osNet) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(Long.valueOf(osNet.segmentId()))
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+
+ nodeService.completeNodes().stream().forEach(node -> {
+ ForwardingObjective.Flag flag = node.type().equals(GATEWAY) ?
+ ForwardingObjective.Flag.VERSATILE :
+ ForwardingObjective.Flag.SPECIFIC;
+
+ RulePopulatorUtil.removeRule(
+ flowObjectiveService,
+ appId,
+ node.intBridge(),
+ sBuilder.build(),
+ flag,
+ ROUTING_RULE_PRIORITY);
+ });
+ }
+
+ private void populateRoutingRules(Host host, Set<OpenstackNetwork> osNets) {
+ String osNetId = host.annotations().value(NETWORK_ID);
+ if (osNetId == null) {
return;
}
- Host host = hostOptional.get();
- Set<IpAddress> ipAddresses = Sets.newHashSet();
- if (action == Action.ASSOCIATE) {
- ipAddresses.add(openstackFloatingIp.floatingIpAddress());
+ DeviceId localDevice = host.location().deviceId();
+ PortNumber localPort = host.location().port();
+ if (!nodeService.dataIp(localDevice).isPresent()) {
+ log.warn("Failed to populate L3 rules");
+ return;
}
- HostDescription hostDescription =
- new DefaultHostDescription(host.mac(), host.vlan(), host.location(), ipAddresses,
- (DefaultAnnotations) host.annotations());
+ // TODO improve pipeline, do we have to install access rules between networks
+ // for every single VMs?
+ osNets.stream().filter(osNet -> !osNet.id().equals(osNetId)).forEach(osNet -> {
+ populateRoutingRulestoSameNode(
+ host.ipAddresses().stream().findFirst().get().getIp4Address(),
+ host.mac(),
+ localPort, localDevice,
+ Long.valueOf(osNet.segmentId()));
- hostProviderService.hostDetected(host.id(), hostDescription, false);
+ nodeService.completeNodes().stream()
+ .filter(node -> node.type().equals(COMPUTE))
+ .filter(node -> !node.intBridge().equals(localDevice))
+ .forEach(node -> populateRoutingRulestoDifferentNode(
+ host.ipAddresses().stream().findFirst().get().getIp4Address(),
+ Long.valueOf(osNet.segmentId()),
+ node.intBridge(),
+ nodeService.dataIp(localDevice).get().getIp4Address()));
+ });
}
- private class InternalHostListener implements HostListener {
-
- private void hostDetected(Host host) {
- String portId = host.annotations().value(Constants.PORT_ID);
- OpenstackPort openstackPort = openstackService.port(portId);
- if (openstackPort == null) {
- log.warn("No OpenstackPort information found from OpenStack for port ID {}", portId);
- return;
- }
-
- Optional<OpenstackPort> routerPort = getRouterInterfacePort(openstackPort.networkId());
- if (routerPort.isPresent()) {
- OpenstackRouterInterface routerInterface = portToRouterInterface(routerPort.get());
- l3EventExecutorService.execute(() ->
- setL3Connection(getOpenstackRouter(routerInterface.id()), openstackPort));
-
- }
+ private void removeRoutingRules(Host host, Set<OpenstackNetwork> osNets) {
+ String osNetId = host.annotations().value(NETWORK_ID);
+ if (osNetId == null) {
+ return;
}
- private void hostRemoved(Host host) {
- String portId = host.annotations().value(Constants.PORT_ID);
- OpenstackPort openstackPort = openstackService.port(portId);
- if (openstackPort == null) {
- log.warn("No OpenstackPort information found from OpenStack for port ID {}", portId);
- return;
- }
+ osNets.stream().filter(osNet -> !osNet.id().equals(osNetId)).forEach(osNet -> {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(host.ipAddresses().stream().findFirst().get().toIpPrefix())
+ .matchTunnelId(Long.valueOf(osNet.segmentId()));
- Optional<OpenstackPort> routerPort = getRouterInterfacePort(openstackPort.networkId());
- if (routerPort.isPresent()) {
- OpenstackRouterInterface routerInterface = portToRouterInterface(routerPort.get());
- IpAddress ipAddress = host.ipAddresses().stream().findFirst().get();
- l3EventExecutorService.execute(() -> rulePopulator.removeL3Rules(ipAddress.getIp4Address(),
- getL3ConnectionList(host.annotations().value(Constants.NETWORK_ID),
- getOpenstackRouterInterface(getOpenstackRouter(routerInterface.id())))));
- }
- }
-
- private boolean isValidHost(Host host) {
- return !host.ipAddresses().isEmpty() &&
- host.annotations().value(Constants.VXLAN_ID) != null &&
- host.annotations().value(Constants.NETWORK_ID) != null &&
- host.annotations().value(Constants.TENANT_ID) != null &&
- host.annotations().value(Constants.PORT_ID) != null;
- }
-
- @Override
- public void event(HostEvent event) {
- Host host = event.subject();
- if (!mastershipService.isLocalMaster(host.location().deviceId())) {
- // do not allow to proceed without mastership
- return;
- }
-
- if (!isValidHost(host)) {
- log.debug("Invalid host event, ignore it {}", host);
- return;
- }
-
- switch (event.type()) {
- case HOST_UPDATED:
- case HOST_ADDED:
- l3EventExecutorService.execute(() -> hostDetected(host));
- break;
- case HOST_REMOVED:
- l3EventExecutorService.execute(() -> hostRemoved(host));
- break;
- default:
- break;
- }
- }
+ nodeService.completeNodes().stream()
+ .filter(node -> node.type().equals(COMPUTE))
+ .forEach(node -> RulePopulatorUtil.removeRule(
+ flowObjectiveService,
+ appId,
+ node.intBridge(),
+ sBuilder.build(),
+ ForwardingObjective.Flag.SPECIFIC,
+ ROUTING_RULE_PRIORITY));
+ });
+ log.debug("Removed routing rule from {} to {}", host, osNets);
}
- private class InternalOpenstackNodeListener implements OpenstackNodeListener {
+ private void populateGatewayToController(long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
- private void nodeComplete() {
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+ tBuilder.setOutput(PortNumber.CONTROLLER);
- rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
- deviceService, driverService, nodeService, gatewayService);
- openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService, hostService,
- openstackService, nodeService, gatewayService);
- openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, nodeService,
- gatewayService);
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
- // Packet handlers must be started AFTER all initialization processes.
- packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
+ gatewayService.getGatewayDeviceIds().stream()
+ .forEach(deviceId -> flowObjectiveService.forward(deviceId, fo));
+ }
- openstackIcmpHandler.requestPacket(appId);
- openstackArpHandler.requestPacket(appId);
+ private void populateCnodeToGateway(long vni) {
+ nodeService.completeNodes().stream()
+ .filter(node -> node.type().equals(COMPUTE))
+ .forEach(node -> populateRuleToGateway(
+ node.intBridge(),
+ gatewayService.getGatewayGroupId(node.intBridge()),
+ vni));
+ }
- openstackService.floatingIps().stream()
- .forEach(f -> floatingIpMap.put(f.id(), f));
+ private void populateRuleToGateway(DeviceId deviceId, GroupId groupId, long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
- reloadInitL3Rules();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+
+ tBuilder.group(groupId);
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void populateRoutingRulestoDifferentNode(Ip4Address vmIp, long vni,
+ DeviceId deviceId, Ip4Address hostIp) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchIPDst(vmIp.toIpPrefix());
+ tBuilder.extension(buildExtension(deviceService, deviceId, hostIp), deviceId)
+ .setOutput(nodeService.tunnelPort(deviceId).get());
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void populateRoutingRulestoSameNode(Ip4Address vmIp, MacAddress vmMac,
+ PortNumber port, DeviceId deviceId, long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(vmIp.toIpPrefix())
+ .matchTunnelId(vni);
+
+ tBuilder.setEthDst(vmMac)
+ .setOutput(port);
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void reloadRoutingRules() {
+ eventExecutor.execute(() -> openstackService.ports().stream()
+ .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+ .forEach(osPort -> {
+ OpenstackRouter osRouter = openstackRouter(osPort.deviceId());
+ setRoutes(osRouter, Optional.empty());
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ setExternalConnection(osRouter, osPort.networkId());
+ }
+ }));
+ }
+
+ @Override
+ protected void hostDetected(Host host) {
+ String osNetId = host.annotations().value(NETWORK_ID);
+ Optional<OpenstackPort> routerIface = routerIfacePort(osNetId);
+ if (!routerIface.isPresent()) {
+ return;
}
+ eventExecutor.execute(() -> setRoutes(
+ openstackRouter(routerIface.get().deviceId()),
+ Optional.of(host)));
+ }
+
+ @Override
+ protected void hostRemoved(Host host) {
+ String osNetId = host.annotations().value(NETWORK_ID);
+ Optional<OpenstackPort> routerIface = routerIfacePort(osNetId);
+ if (!routerIface.isPresent()) {
+ return;
+ }
+ Set<OpenstackNetwork> routableNets = routableNetworks(routerIface.get().deviceId());
+ eventExecutor.execute(() -> removeRoutingRules(host, routableNets));
+ }
+
+ private class InternalNodeListener implements OpenstackNodeListener {
@Override
public void event(OpenstackNodeEvent event) {
@@ -730,28 +481,22 @@
switch (event.type()) {
case COMPLETE:
log.info("COMPLETE node {} detected", node.hostname());
- l3EventExecutorService.execute(() -> nodeComplete());
+ if (node.type() == GATEWAY) {
+ GatewayNode gnode = GatewayNode.builder()
+ .gatewayDeviceId(node.intBridge())
+ .dataIpAddress(node.dataIp().getIp4Address())
+ .uplinkIntf(node.externalPortName().get())
+ .build();
+ gatewayService.addGatewayNode(gnode);
+ }
+ eventExecutor.execute(OpenstackRoutingManager.this::reloadRoutingRules);
break;
+ case INIT:
+ case DEVICE_CREATED:
case INCOMPLETE:
- break;
default:
break;
}
}
}
-
- private class InternalHostProvider extends AbstractProvider implements HostProvider {
-
- /**
- * Creates a provider with the supplier identifier.
- */
- protected InternalHostProvider() {
- super(PID);
- }
-
- @Override
- public void triggerProbe(Host host) {
- // nothing to do
- }
- }
}