Refactored OpenstackRouting to support multiple gateway nodes
Change-Id: I6870ca9a4fd6f6b1cf2d2be72f52ef87827e1d2c
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java
deleted file mode 100644
index 3d1c013..0000000
--- a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2016-present Open Networking Laboratory
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.onosproject.openstacknetworking.routing;
-
-import org.onosproject.net.Host;
-import org.onosproject.openstackinterface.OpenstackFloatingIP;
-
-/**
- * Handle FloatingIP Event for Managing Flow Rules In Openstack Nodes.
- */
-public class OpenstackFloatingIPHandler implements Runnable {
-
- public enum Action {
- ASSOCIATE,
- DISSASSOCIATE
- }
-
- private final OpenstackFloatingIP floatingIP;
- private final OpenstackRoutingRulePopulator rulePopulator;
- private final Host host;
- private final Action action;
-
-
- OpenstackFloatingIPHandler(OpenstackRoutingRulePopulator rulePopulator,
- OpenstackFloatingIP openstackFloatingIP, Action action, Host host) {
- this.floatingIP = openstackFloatingIP;
- this.rulePopulator = rulePopulator;
- this.action = action;
- this.host = host;
- }
-
- @Override
- public void run() {
- if (action == Action.ASSOCIATE) {
- rulePopulator.populateFloatingIpRules(floatingIP);
- } else {
- rulePopulator.removeFloatingIpRules(floatingIP, host);
- }
- }
-}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIpManager.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIpManager.java
new file mode 100644
index 0000000..ba60fe8
--- /dev/null
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIpManager.java
@@ -0,0 +1,329 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import com.google.common.base.Strings;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IpAddress;
+import org.onlab.util.KryoNamespace;
+import org.onlab.util.Tools;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.net.host.HostService;
+import org.onosproject.openstackinterface.OpenstackFloatingIP;
+import org.onosproject.openstacknetworking.Constants;
+import org.onosproject.openstacknetworking.OpenstackFloatingIpService;
+import org.onosproject.openstacknetworking.RulePopulatorUtil;
+import org.onosproject.openstacknode.OpenstackNode;
+import org.onosproject.openstacknode.OpenstackNodeEvent;
+import org.onosproject.openstacknode.OpenstackNodeListener;
+import org.onosproject.openstacknode.OpenstackNodeService;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.Versioned;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.onosproject.openstacknetworking.RulePopulatorUtil.buildExtension;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY;
+
+
+@Service
+@Component(immediate = true)
+public class OpenstackFloatingIpManager implements OpenstackFloatingIpService {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected HostService hostService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected FlowObjectiveService flowObjectiveService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected StorageService storageService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackNodeService nodeService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ScalableGatewayService gatewayService;
+
+ private static final String NOT_ASSOCIATED = "null";
+ private static final KryoNamespace.Builder FLOATING_IP_SERIALIZER =
+ KryoNamespace.newBuilder().register(KryoNamespaces.API);
+
+ private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+ private final InternalNodeListener nodeListener = new InternalNodeListener();
+ private ConsistentMap<IpAddress, Host> floatingIpMap;
+
+ private ApplicationId appId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(ROUTING_APP_ID);
+ nodeService.addListener(nodeListener);
+ floatingIpMap = storageService.<IpAddress, Host>consistentMapBuilder()
+ .withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
+ .withName("openstackrouting-floatingip")
+ .withApplicationId(appId)
+ .build();
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ nodeService.removeListener(nodeListener);
+ log.info("Stopped");
+ }
+
+ @Override
+ public void createFloatingIp(OpenstackFloatingIP floatingIp) {
+ }
+
+ @Override
+ public void updateFloatingIp(OpenstackFloatingIP floatingIp) {
+ if (Strings.isNullOrEmpty(floatingIp.portId()) ||
+ floatingIp.portId().equals(NOT_ASSOCIATED)) {
+ eventExecutor.execute(() -> disassociateFloatingIp(floatingIp));
+ } else {
+ eventExecutor.execute(() -> associateFloatingIp(floatingIp));
+ }
+ }
+
+ @Override
+ public void deleteFloatingIp(String floatingIpId) {
+ }
+
+ private void associateFloatingIp(OpenstackFloatingIP floatingIp) {
+ Optional<Host> associatedVm = Tools.stream(hostService.getHosts())
+ .filter(host -> Objects.equals(
+ host.annotations().value(PORT_ID),
+ floatingIp.portId()))
+ .findAny();
+ if (!associatedVm.isPresent()) {
+ log.warn("Failed to associate floating IP({}) to port:{}",
+ floatingIp.floatingIpAddress(),
+ floatingIp.portId());
+ return;
+ }
+
+ floatingIpMap.put(floatingIp.floatingIpAddress(), associatedVm.get());
+ populateFloatingIpRules(floatingIp.floatingIpAddress(), associatedVm.get());
+
+ log.info("Associated floating IP {} to fixed IP {}",
+ floatingIp.floatingIpAddress(), floatingIp.fixedIpAddress());
+ }
+
+ private void disassociateFloatingIp(OpenstackFloatingIP floatingIp) {
+ Versioned<Host> associatedVm = floatingIpMap.remove(floatingIp.floatingIpAddress());
+ if (associatedVm == null) {
+ log.warn("Failed to disassociate floating IP({})",
+ floatingIp.floatingIpAddress());
+ // No VM is actually associated with the floating IP, do nothing
+ return;
+ }
+
+ removeFloatingIpRules(floatingIp.floatingIpAddress(), associatedVm.value());
+ log.info("Disassociated floating IP {} from fixed IP {}",
+ floatingIp.floatingIpAddress(),
+ associatedVm.value().ipAddresses());
+ }
+
+ private void populateFloatingIpRules(IpAddress floatingIp, Host associatedVm) {
+ populateFloatingIpIncomingRules(floatingIp, associatedVm);
+ populateFloatingIpOutgoingRules(floatingIp, associatedVm);
+ }
+
+ private void removeFloatingIpRules(IpAddress floatingIp, Host associatedVm) {
+ Optional<IpAddress> fixedIp = associatedVm.ipAddresses().stream().findFirst();
+ if (!fixedIp.isPresent()) {
+ log.warn("Failed to remove floating IP({}) from {}",
+ floatingIp, associatedVm);
+ return;
+ }
+
+ TrafficSelector.Builder sOutgoingBuilder = DefaultTrafficSelector.builder();
+ TrafficSelector.Builder sIncomingBuilder = DefaultTrafficSelector.builder();
+
+ sOutgoingBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(Long.valueOf(associatedVm.annotations().value(VXLAN_ID)))
+ .matchIPSrc(fixedIp.get().toIpPrefix());
+
+ sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(floatingIp.toIpPrefix());
+
+ gatewayService.getGatewayDeviceIds().stream().forEach(deviceId -> {
+ RulePopulatorUtil.removeRule(
+ flowObjectiveService,
+ appId,
+ deviceId,
+ sOutgoingBuilder.build(),
+ ForwardingObjective.Flag.VERSATILE,
+ FLOATING_RULE_PRIORITY);
+
+ RulePopulatorUtil.removeRule(
+ flowObjectiveService,
+ appId,
+ deviceId,
+ sIncomingBuilder.build(),
+ ForwardingObjective.Flag.VERSATILE,
+ FLOATING_RULE_PRIORITY);
+ });
+ }
+
+ private void populateFloatingIpIncomingRules(IpAddress floatingIp, Host associatedVm) {
+ DeviceId cnodeId = associatedVm.location().deviceId();
+ Optional<IpAddress> dataIp = nodeService.dataIp(cnodeId);
+ Optional<IpAddress> fixedIp = associatedVm.ipAddresses().stream().findFirst();
+
+ if (!fixedIp.isPresent() || !dataIp.isPresent()) {
+ log.warn("Failed to associate floating IP({})", floatingIp);
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(floatingIp.toIpPrefix())
+ .build();
+
+ gatewayService.getGatewayDeviceIds().stream().forEach(gnodeId -> {
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setEthSrc(Constants.DEFAULT_GATEWAY_MAC)
+ .setEthDst(associatedVm.mac())
+ .setIpDst(associatedVm.ipAddresses().stream().findFirst().get())
+ .setTunnelId(Long.valueOf(associatedVm.annotations().value(VXLAN_ID)))
+ .extension(buildExtension(deviceService, cnodeId, dataIp.get().getIp4Address()),
+ cnodeId)
+ .setOutput(nodeService.tunnelPort(gnodeId).get())
+ .build();
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(FLOATING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(gnodeId, fo);
+ });
+ }
+
+ private void populateFloatingIpOutgoingRules(IpAddress floatingIp, Host associatedVm) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(Long.valueOf(associatedVm.annotations().value(VXLAN_ID)))
+ .matchIPSrc(associatedVm.ipAddresses().stream().findFirst().get().toIpPrefix())
+ .build();
+
+ gatewayService.getGatewayDeviceIds().stream().forEach(gnodeId -> {
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setIpSrc(floatingIp)
+ .setEthSrc(Constants.DEFAULT_GATEWAY_MAC)
+ .setEthDst(Constants.DEFAULT_EXTERNAL_ROUTER_MAC)
+ .setOutput(gatewayService.getUplinkPort(gnodeId))
+ .build();
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(FLOATING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(gnodeId, fo);
+ });
+ }
+
+ private void reloadFloatingIpRules() {
+ floatingIpMap.entrySet().stream().forEach(entry -> {
+ IpAddress floatingIp = entry.getKey();
+ Host associatedVm = entry.getValue().value();
+
+ populateFloatingIpRules(floatingIp, associatedVm);
+ log.debug("Reload floating IP {} mapped to {}",
+ floatingIp, associatedVm.ipAddresses());
+ });
+ }
+
+ // TODO apply existing floating IPs on service start-up by handling host event
+ // TODO consider the case that port with associated floating IP is attached to a VM
+
+ private class InternalNodeListener implements OpenstackNodeListener {
+
+ @Override
+ public void event(OpenstackNodeEvent event) {
+ OpenstackNode node = event.node();
+
+ switch (event.type()) {
+ case COMPLETE:
+ if (node.type() == GATEWAY) {
+ log.info("GATEWAY node {} detected", node.hostname());
+ GatewayNode gnode = GatewayNode.builder()
+ .gatewayDeviceId(node.intBridge())
+ .dataIpAddress(node.dataIp().getIp4Address())
+ .uplinkIntf(node.externalPortName().get())
+ .build();
+ gatewayService.addGatewayNode(gnode);
+ eventExecutor.execute(OpenstackFloatingIpManager.this::reloadFloatingIpRules);
+ }
+ break;
+ case INIT:
+ case DEVICE_CREATED:
+ case INCOMPLETE:
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
index 7db84bb..b6bf09a 100644
--- a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
@@ -16,322 +16,319 @@
package org.onosproject.openstacknetworking.routing;
import com.google.common.collect.Maps;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.onlab.packet.Ethernet;
import org.onlab.packet.ICMP;
import org.onlab.packet.IPv4;
import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
-import org.onlab.packet.MacAddress;
import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Host;
-import org.onosproject.net.Port;
-import org.onosproject.net.PortNumber;
-import org.onosproject.net.device.DeviceService;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.host.HostService;
import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.InboundPacket;
import org.onosproject.net.packet.OutboundPacket;
import org.onosproject.net.packet.PacketContext;
import org.onosproject.net.packet.PacketPriority;
+import org.onosproject.net.packet.PacketProcessor;
import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstacknetworking.Constants;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstacknode.OpenstackNode;
+import org.onosproject.openstacknode.OpenstackNodeEvent;
+import org.onosproject.openstacknode.OpenstackNodeListener;
import org.onosproject.openstacknode.OpenstackNodeService;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
+import java.util.concurrent.ExecutorService;
-import static com.google.common.base.Preconditions.checkNotNull;
-
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY;
import static org.slf4j.LoggerFactory.getLogger;
/**
- * Handle ICMP packet sent from Openstack Gateway nodes.
+ * Handle ICMP packet sent from OpenStack Gateway nodes.
+ * For a request to any private network gateway IPs, it generates fake reply.
+ * For a request to the external network, it does source NAT with a public IP and
+ * forward the request to the external only if the request instance has external
+ * connection setups.
*/
+@Component(immediate = true)
public class OpenstackIcmpHandler {
protected final Logger log = getLogger(getClass());
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CoreService coreService;
- private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
- private static final String PORTNAME = "portName";
- private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
- private static final String NETWORK_FLOATING_IP = "network:floatingip";
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected PacketService packetService;
- private final PacketService packetService;
- private final DeviceService deviceService;
- private final ScalableGatewayService gatewayService;
- private final HostService hostService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected HostService hostService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackInterfaceService openstackService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ScalableGatewayService gatewayService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackNodeService nodeService;
+
+ private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+ private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
+ private final InternalNodeListener nodeListener = new InternalNodeListener();
private final Map<String, Host> icmpInfoMap = Maps.newHashMap();
- private final OpenstackInterfaceService openstackService;
- private final OpenstackNodeService nodeService;
- /**
- * Default constructor.
- *
- * @param packetService packet service
- * @param deviceService device service
- * @param openstackService openstackInterface service
- */
- OpenstackIcmpHandler(PacketService packetService,
- DeviceService deviceService,
- HostService hostService,
- OpenstackInterfaceService openstackService,
- OpenstackNodeService nodeService,
- ScalableGatewayService gatewayService
- ) {
- this.packetService = packetService;
- this.deviceService = deviceService;
- this.hostService = hostService;
- this.openstackService = checkNotNull(openstackService);
- this.nodeService = nodeService;
- this.gatewayService = gatewayService;
+ ApplicationId appId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(ROUTING_APP_ID);
+ packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
+ nodeService.addListener(nodeListener);
+ requestPacket(appId);
+
+ log.info("Started");
}
- /**
- * Requests ICMP packet.
- *
- * @param appId Application Id
- */
- public void requestPacket(ApplicationId appId) {
+ @Deactivate
+ protected void deactivate() {
+ packetService.removeProcessor(packetProcessor);
+ log.info("Stopped");
+ }
+
+ private void requestPacket(ApplicationId appId) {
TrafficSelector icmpSelector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
.matchIPProtocol(IPv4.PROTOCOL_ICMP)
.build();
- // TODO: Return the correct gateway node
- Optional<OpenstackNode> gwNode = nodeService.nodes().stream()
- .filter(n -> n.type().equals(OpenstackNodeService.NodeType.GATEWAY))
- .findFirst();
-
- if (!gwNode.isPresent()) {
- log.warn("No Gateway is defined.");
- return;
- }
-
- packetService.requestPackets(icmpSelector,
- PacketPriority.CONTROL,
- appId,
- Optional.of(gwNode.get().intBridge()));
+ gatewayService.getGatewayDeviceIds().stream().forEach(gateway -> {
+ packetService.requestPackets(icmpSelector,
+ PacketPriority.CONTROL,
+ appId,
+ Optional.of(gateway));
+ log.debug("Requested ICMP packet on {}", gateway);
+ });
}
- /**
- * Handles ICMP packet.
- *
- * @param context packet context
- * @param ethernet ethernet
- */
- public void processIcmpPacket(PacketContext context, Ethernet ethernet) {
- checkNotNull(context, "context can not be null");
- checkNotNull(ethernet, "ethernet can not be null");
-
+ private void processIcmpPacket(PacketContext context, Ethernet ethernet) {
IPv4 ipPacket = (IPv4) ethernet.getPayload();
-
- log.debug("icmpEvent called from ip {}, mac {}", Ip4Address.valueOf(ipPacket.getSourceAddress()).toString(),
- ethernet.getSourceMAC().toString());
+ log.trace("Processing ICMP packet from ip {}, mac {}",
+ Ip4Address.valueOf(ipPacket.getSourceAddress()),
+ ethernet.getSourceMAC());
ICMP icmp = (ICMP) ipPacket.getPayload();
short icmpId = getIcmpId(icmp);
- DeviceId deviceId = context.inPacket().receivedFrom().deviceId();
- PortNumber portNumber = context.inPacket().receivedFrom().port();
- if (icmp.getIcmpType() == ICMP.TYPE_ECHO_REQUEST) {
- //TODO: Considers icmp between internal subnets which are belonged to the same router.
- Optional<Host> host = hostService.getHostsByMac(ethernet.getSourceMAC()).stream().findFirst();
- if (!host.isPresent()) {
- log.warn("No host found for MAC {}", ethernet.getSourceMAC());
- return;
- }
+ DeviceId srcDevice = context.inPacket().receivedFrom().deviceId();
+ switch (icmp.getIcmpType()) {
+ case ICMP.TYPE_ECHO_REQUEST:
+ Optional<Host> reqHost = hostService.getHostsByMac(ethernet.getSourceMAC())
+ .stream().findFirst();
+ if (!reqHost.isPresent()) {
+ log.warn("No host found for MAC {}", ethernet.getSourceMAC());
+ return;
+ }
- IpAddress gatewayIp = IpAddress.valueOf(host.get().annotations().value(Constants.GATEWAY_IP));
- if (ipPacket.getDestinationAddress() == gatewayIp.getIp4Address().toInt()) {
- processIcmpPacketSentToGateway(ipPacket, icmp, host.get());
- } else {
- Ip4Address pNatIpAddress = pNatIpForPort(host.get());
- checkNotNull(pNatIpAddress, "pNatIpAddress can not be null");
+ // TODO Considers icmp between internal subnets belong to the same router.
+ // TODO do we have to support ICMP reply for non-existing gateway?
+ Ip4Address gatewayIp = Ip4Address.valueOf(
+ reqHost.get().annotations().value(Constants.GATEWAY_IP));
+ if (Objects.equals(ipPacket.getDestinationAddress(), gatewayIp.toInt())) {
+ processRequestToGateway(ipPacket, reqHost.get());
+ } else {
+ Optional<Ip4Address> srcNatIp = getSrcNatIp(reqHost.get());
+ if (!srcNatIp.isPresent()) {
+ log.trace("VM {} has no external connection", reqHost.get());
+ return;
+ }
- sendRequestPacketToExt(ipPacket, icmp, deviceId, pNatIpAddress);
-
+ sendRequestToExternal(ipPacket, srcDevice, srcNatIp.get());
+ String icmpInfoKey = String.valueOf(icmpId)
+ .concat(String.valueOf(srcNatIp.get().toInt()))
+ .concat(String.valueOf(ipPacket.getDestinationAddress()));
+ icmpInfoMap.putIfAbsent(icmpInfoKey, reqHost.get());
+ }
+ break;
+ case ICMP.TYPE_ECHO_REPLY:
String icmpInfoKey = String.valueOf(icmpId)
- .concat(String.valueOf(pNatIpAddress.toInt()))
- .concat(String.valueOf(ipPacket.getDestinationAddress()));
- icmpInfoMap.putIfAbsent(icmpInfoKey, host.get());
- }
- } else if (icmp.getIcmpType() == ICMP.TYPE_ECHO_REPLY) {
- String icmpInfoKey = String.valueOf(icmpId)
- .concat(String.valueOf(ipPacket.getDestinationAddress()))
- .concat(String.valueOf(ipPacket.getSourceAddress()));
+ .concat(String.valueOf(ipPacket.getDestinationAddress()))
+ .concat(String.valueOf(ipPacket.getSourceAddress()));
- processResponsePacketFromExternalToHost(ipPacket, icmp, icmpInfoMap.get(icmpInfoKey));
-
- icmpInfoMap.remove(icmpInfoKey);
+ processReplyFromExternal(ipPacket, icmpInfoMap.get(icmpInfoKey));
+ icmpInfoMap.remove(icmpInfoKey);
+ break;
+ default:
+ break;
}
}
- private void processIcmpPacketSentToExtenal(IPv4 icmpRequestIpv4, ICMP icmpRequest,
- int destAddr, MacAddress destMac,
- DeviceId deviceId, PortNumber portNumber) {
- icmpRequest.setChecksum((short) 0);
- icmpRequest.setIcmpType(ICMP.TYPE_ECHO_REPLY).resetChecksum();
- icmpRequestIpv4.setSourceAddress(icmpRequestIpv4.getDestinationAddress())
- .setDestinationAddress(destAddr).resetChecksum();
- icmpRequestIpv4.setPayload(icmpRequest);
- Ethernet icmpResponseEth = new Ethernet();
- icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
- // TODO: Get the correct GW MAC
- .setSourceMACAddress(Constants.GW_EXT_INT_MAC)
- .setDestinationMACAddress(destMac).setPayload(icmpRequestIpv4);
- TrafficTreatment treatment = DefaultTrafficTreatment.builder().setOutput(portNumber).build();
- OutboundPacket packet = new DefaultOutboundPacket(deviceId,
- treatment, ByteBuffer.wrap(icmpResponseEth.serialize()));
- packetService.emit(packet);
+ // TODO do we have to handle the request to the fake gateway?
+ private void processRequestToGateway(IPv4 ipPacket, Host reqHost) {
+ ICMP icmpReq = (ICMP) ipPacket.getPayload();
+ icmpReq.setChecksum((short) 0);
+ icmpReq.setIcmpType(ICMP.TYPE_ECHO_REPLY).resetChecksum();
+
+ ipPacket.setSourceAddress(ipPacket.getDestinationAddress())
+ .setDestinationAddress(ipPacket.getSourceAddress())
+ .resetChecksum();
+
+ ipPacket.setPayload(icmpReq);
+ Ethernet icmpReply = new Ethernet();
+ icmpReply.setEtherType(Ethernet.TYPE_IPV4)
+ .setSourceMACAddress(Constants.DEFAULT_GATEWAY_MAC)
+ .setDestinationMACAddress(reqHost.mac())
+ .setPayload(icmpReq);
+
+ sendReply(icmpReply, reqHost);
}
- private void processIcmpPacketSentToGateway(IPv4 icmpRequestIpv4, ICMP icmpRequest,
- Host host) {
- icmpRequest.setChecksum((short) 0);
- icmpRequest.setIcmpType(ICMP.TYPE_ECHO_REPLY)
- .resetChecksum();
-
- Ip4Address ipAddress = host.ipAddresses().stream().findAny().get().getIp4Address();
- icmpRequestIpv4.setSourceAddress(icmpRequestIpv4.getDestinationAddress())
- .setDestinationAddress(ipAddress.toInt())
- .resetChecksum();
-
- icmpRequestIpv4.setPayload(icmpRequest);
-
- Ethernet icmpResponseEth = new Ethernet();
-
- icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
- .setSourceMACAddress(Constants.GATEWAY_MAC)
- .setDestinationMACAddress(host.mac())
- .setPayload(icmpRequestIpv4);
-
- sendResponsePacketToHost(icmpResponseEth, host);
- }
-
- private void sendRequestPacketToExt(IPv4 icmpRequestIpv4, ICMP icmpRequest, DeviceId deviceId,
- Ip4Address pNatIpAddress) {
- icmpRequest.resetChecksum();
- icmpRequestIpv4.setSourceAddress(pNatIpAddress.toInt())
- .resetChecksum();
- icmpRequestIpv4.setPayload(icmpRequest);
+ private void sendRequestToExternal(IPv4 ipPacket, DeviceId srcDevice, Ip4Address srcNatIp) {
+ ICMP icmpReq = (ICMP) ipPacket.getPayload();
+ icmpReq.resetChecksum();
+ ipPacket.setSourceAddress(srcNatIp.toInt()).resetChecksum();
+ ipPacket.setPayload(icmpReq);
Ethernet icmpRequestEth = new Ethernet();
-
icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
- // TODO: Get the correct one - Scalable Gateway ...
- .setSourceMACAddress(Constants.GW_EXT_INT_MAC)
- .setDestinationMACAddress(Constants.PHY_ROUTER_MAC)
- .setPayload(icmpRequestIpv4);
-
- // TODO: Return the correct gateway node
- Optional<OpenstackNode> gwNode = nodeService.nodes().stream()
- .filter(n -> n.type().equals(OpenstackNodeService.NodeType.GATEWAY))
- .findFirst();
-
- if (!gwNode.isPresent()) {
- log.warn("No Gateway is defined.");
- return;
- }
+ .setSourceMACAddress(DEFAULT_GATEWAY_MAC)
+ .setDestinationMACAddress(DEFAULT_EXTERNAL_ROUTER_MAC)
+ .setPayload(ipPacket);
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- // FIXME: please double check this.
- .setOutput(getPortForAnnotationPortName(gwNode.get().intBridge(),
- // FIXME: please double check this.
- org.onosproject.openstacknode.Constants.PATCH_INTG_BRIDGE))
+ .setOutput(gatewayService.getUplinkPort(srcDevice))
.build();
- OutboundPacket packet = new DefaultOutboundPacket(deviceId,
- treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
+ OutboundPacket packet = new DefaultOutboundPacket(
+ srcDevice,
+ treatment,
+ ByteBuffer.wrap(icmpRequestEth.serialize()));
packetService.emit(packet);
}
- private void processResponsePacketFromExternalToHost(IPv4 icmpResponseIpv4, ICMP icmpResponse,
- Host host) {
- icmpResponse.resetChecksum();
+ private void processReplyFromExternal(IPv4 ipPacket, Host dstHost) {
+ ICMP icmpReply = (ICMP) ipPacket.getPayload();
+ icmpReply.resetChecksum();
- Ip4Address ipAddress = host.ipAddresses().stream().findFirst().get().getIp4Address();
- icmpResponseIpv4.setDestinationAddress(ipAddress.toInt())
+ Ip4Address ipAddress = dstHost.ipAddresses().stream().findFirst().get().getIp4Address();
+ ipPacket.setDestinationAddress(ipAddress.toInt())
.resetChecksum();
- icmpResponseIpv4.setPayload(icmpResponse);
+ ipPacket.setPayload(icmpReply);
Ethernet icmpResponseEth = new Ethernet();
-
icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
- .setSourceMACAddress(Constants.GATEWAY_MAC)
- .setDestinationMACAddress(host.mac())
- .setPayload(icmpResponseIpv4);
+ .setSourceMACAddress(Constants.DEFAULT_GATEWAY_MAC)
+ .setDestinationMACAddress(dstHost.mac())
+ .setPayload(ipPacket);
- sendResponsePacketToHost(icmpResponseEth, host);
+ sendReply(icmpResponseEth, dstHost);
}
- private void sendResponsePacketToHost(Ethernet icmpResponseEth, Host host) {
-
+ private void sendReply(Ethernet icmpReply, Host dstHost) {
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- .setOutput(host.location().port())
+ .setOutput(dstHost.location().port())
.build();
- OutboundPacket packet = new DefaultOutboundPacket(host.location().deviceId(),
- treatment, ByteBuffer.wrap(icmpResponseEth.serialize()));
+ OutboundPacket packet = new DefaultOutboundPacket(
+ dstHost.location().deviceId(),
+ treatment,
+ ByteBuffer.wrap(icmpReply.serialize()));
packetService.emit(packet);
}
+ private Optional<Ip4Address> getSrcNatIp(Host host) {
+ // TODO cache external gateway IP for each network because
+ // asking Neutron for every ICMP request is a bad idea
+ Optional<OpenstackPort> osPort = openstackService.ports().stream()
+ .filter(port -> port.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ Objects.equals(host.annotations().value(NETWORK_ID),
+ port.networkId()))
+ .findAny();
+ if (!osPort.isPresent()) {
+ return Optional.empty();
+ }
+
+ OpenstackRouter osRouter = openstackService.router(osPort.get().deviceId());
+ if (osRouter == null) {
+ return Optional.empty();
+ }
+
+ return osRouter.gatewayExternalInfo().externalFixedIps()
+ .values().stream().findAny();
+ }
+
private short getIcmpId(ICMP icmp) {
return ByteBuffer.wrap(icmp.serialize(), 4, 2).getShort();
}
- private Ip4Address pNatIpForPort(Host host) {
+ private class InternalPacketProcessor implements PacketProcessor {
- OpenstackPort openstackPort = openstackService.ports().stream()
- .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_INTERFACE) &&
- p.networkId().equals(host.annotations().value(Constants.NETWORK_ID)))
- .findAny().orElse(null);
+ @Override
+ public void process(PacketContext context) {
+ if (context.isHandled()) {
+ return;
+ } else if (!gatewayService.getGatewayDeviceIds().contains(
+ context.inPacket().receivedFrom().deviceId())) {
+ // return if the packet is not from gateway nodes
+ return;
+ }
- checkNotNull(openstackPort, "openstackPort can not be null");
+ InboundPacket pkt = context.inPacket();
+ Ethernet ethernet = pkt.parsed();
+ if (ethernet == null || ethernet.getEtherType() == Ethernet.TYPE_ARP) {
+ return;
+ }
- return openstackService.router(openstackPort.deviceId())
- .gatewayExternalInfo().externalFixedIps().values()
- .stream().findAny().orElse(null);
- }
-
- private PortNumber getPortForAnnotationPortName(DeviceId deviceId, String match) {
- Port port = deviceService.getPorts(deviceId).stream()
- .filter(p -> p.annotations().value(PORTNAME).equals(match))
- .findAny().orElse(null);
-
- checkNotNull(port, "port cannot be null");
-
- return port.number();
- }
-
- private boolean requestToOpenstackRoutingNetwork(int destAddr) {
- OpenstackPort port = openstackService.ports().stream()
- .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_GATEWAY) ||
- p.deviceOwner().equals(NETWORK_FLOATING_IP))
- .filter(p -> p.fixedIps().containsValue(
- Ip4Address.valueOf(destAddr)))
- .findAny().orElse(null);
- if (port == null) {
- return false;
+ IPv4 iPacket = (IPv4) ethernet.getPayload();
+ if (iPacket.getProtocol() == IPv4.PROTOCOL_ICMP) {
+ eventExecutor.execute(() -> processIcmpPacket(context, ethernet));
+ }
}
- return true;
}
- private Map<DeviceId, PortNumber> getExternalInfo() {
- Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
- gatewayService.getGatewayDeviceIds().forEach(deviceId ->
- externalInfoMap.putIfAbsent(deviceId, gatewayService.getUplinkPort(deviceId)));
- return externalInfoMap;
+
+ private class InternalNodeListener implements OpenstackNodeListener {
+
+ @Override
+ public void event(OpenstackNodeEvent event) {
+ OpenstackNode node = event.node();
+
+ switch (event.type()) {
+ case COMPLETE:
+ if (node.type() == GATEWAY) {
+ log.info("GATEWAY node {} detected", node.hostname());
+ eventExecutor.execute(() -> requestPacket(appId));
+ }
+ break;
+ case INIT:
+ case DEVICE_CREATED:
+ case INCOMPLETE:
+ default:
+ break;
+ }
+ }
}
}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
index 5af208d..3b40aee 100644
--- a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
@@ -15,109 +15,156 @@
*/
package org.onosproject.openstacknetworking.routing;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.onlab.packet.Ethernet;
import org.onlab.packet.IPv4;
import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
import org.onlab.packet.MacAddress;
import org.onlab.packet.TCP;
+import org.onlab.packet.TpPort;
import org.onlab.packet.UDP;
+import org.onlab.util.KryoNamespace;
+import org.onlab.util.Tools;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
import org.onosproject.net.DeviceId;
-import org.onosproject.net.Port;
+import org.onosproject.net.Host;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.net.host.HostService;
import org.onosproject.net.packet.DefaultOutboundPacket;
import org.onosproject.net.packet.InboundPacket;
import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketProcessor;
import org.onosproject.net.packet.PacketService;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstackinterface.OpenstackRouter;
-import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.openstacknetworking.RulePopulatorUtil;
+import org.onosproject.openstacknode.OpenstackNodeService;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.onlab.osgi.DefaultServiceDirectory.getService;
-
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.slf4j.LoggerFactory.getLogger;
/**
- * Handle NAT packet processing for Managing Flow Rules In Openstack Nodes.
+ * Handle NAT packet processing for managing flow rules in openstack nodes.
*/
-public class OpenstackPnatHandler implements Runnable {
+@Component(immediate = true)
+public class OpenstackPnatHandler {
+ private final Logger log = getLogger(getClass());
- volatile PacketContext context;
- private final Logger log = LoggerFactory.getLogger(getClass());
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CoreService coreService;
- private final OpenstackRoutingRulePopulator rulePopulator;
- private final int portNum;
- private final OpenstackPort openstackPort;
- private final Port port;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected PacketService packetService;
- private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
- private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected HostService hostService;
- OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
- int portNum, OpenstackPort openstackPort, Port port) {
- this.rulePopulator = checkNotNull(rulePopulator);
- this.context = checkNotNull(context);
- this.portNum = checkNotNull(portNum);
- this.openstackPort = checkNotNull(openstackPort);
- this.port = checkNotNull(port);
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected StorageService storageService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected FlowObjectiveService flowObjectiveService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackInterfaceService openstackService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackNodeService nodeService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ScalableGatewayService gatewayService;
+
+ private static final KryoNamespace.Builder NUMBER_SERIALIZER = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API);
+
+ private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
+ private static final int TP_PORT_MINIMUM_NUM = 1024;
+ private static final int TP_PORT_MAXIMUM_NUM = 65535;
+
+ private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+ private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
+
+ private ConsistentMap<Integer, String> tpPortNumMap;
+ private ApplicationId appId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(ROUTING_APP_ID);
+ tpPortNumMap = storageService.<Integer, String>consistentMapBuilder()
+ .withSerializer(Serializer.using(NUMBER_SERIALIZER.build()))
+ .withName("openstackrouting-tpportnum")
+ .withApplicationId(appId)
+ .build();
+
+ packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
+ log.info("Started");
}
- @Override
- public void run() {
- InboundPacket inboundPacket = context.inPacket();
- Ethernet ethernet = checkNotNull(inboundPacket.parsed());
+ @Deactivate
+ protected void deactivate() {
+ packetService.removeProcessor(packetProcessor);
+ log.info("Stopped");
+ }
- //TODO: Considers IPV6
- if (ethernet.getEtherType() != Ethernet.TYPE_IPV4) {
- log.warn("Now, we just consider IP version 4");
+ private void processPnatPacket(PacketContext context, Ethernet ethernet) {
+ IPv4 iPacket = (IPv4) ethernet.getPayload();
+ InboundPacket inboundPacket = context.inPacket();
+
+ int srcPort = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
+ OpenstackPort osPort = getOpenstackPort(ethernet.getSourceMAC());
+ if (osPort == null) {
+ return;
+ }
+ Ip4Address externalGatewayIp = getExternalGatewayIp(osPort);
+ if (externalGatewayIp == null) {
return;
}
- OpenstackRouter router = getOpenstackRouter(openstackPort);
+ populatePnatFlowRules(context.inPacket(),
+ osPort,
+ TpPort.tpPort(srcPort),
+ externalGatewayIp);
- MacAddress externalMac = MacAddress.NONE;
- MacAddress routerMac = MacAddress.NONE;
-
- rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
- getExternalIp(router), externalMac, routerMac);
-
- packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
+ packetOut((Ethernet) ethernet.clone(),
+ inboundPacket.receivedFrom().deviceId(),
+ srcPort,
+ externalGatewayIp);
}
- private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
- OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
-
- OpenstackPort port = networkingService.ports()
- .stream()
- .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
- .filter(p -> checkSameSubnet(p, openstackPort))
- .findAny()
- .orElse(null);
-
- return checkNotNull(networkingService.router(port.deviceId()));
- }
-
- private boolean checkSameSubnet(OpenstackPort p, OpenstackPort openstackPort) {
- String key1 = checkNotNull(p.fixedIps().keySet().stream().findFirst().orElse(null)).toString();
- String key2 = checkNotNull(openstackPort.fixedIps().keySet().stream().findFirst().orElse(null)).toString();
- return key1.equals(key2) ? true : false;
- }
-
- private Ip4Address getExternalIp(OpenstackRouter router) {
- return router.gatewayExternalInfo().externalFixedIps().values().stream().findAny().orElse(null);
- }
-
- private void packetOut(Ethernet ethernet, DeviceId deviceId, int portNum, OpenstackRouter router) {
- PacketService packetService = getService(PacketService.class);
-
+ private void packetOut(Ethernet ethernet, DeviceId deviceId, int portNum, Ip4Address externalIp) {
IPv4 iPacket = (IPv4) ethernet.getPayload();
-
TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
switch (iPacket.getProtocol()) {
@@ -136,26 +183,251 @@
iPacket.setPayload(udpPacket);
break;
default:
- log.error("Temporally, this method can process UDP and TCP protocol.");
+ log.trace("Temporally, this method can process UDP and TCP protocol.");
return;
}
- iPacket.setSourceAddress(getExternalIp(router).toString());
+ iPacket.setSourceAddress(externalIp.toString());
iPacket.resetChecksum();
iPacket.setParent(ethernet);
ethernet.setPayload(iPacket);
- ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
- GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
- if (gatewayNode.getUplinkIntf() == null) {
- log.error(EXTERNAL_PORT_NULL, deviceId.toString());
+ treatment.setOutput(gatewayService.getUplinkPort(deviceId));
+ ethernet.resetChecksum();
+ packetService.emit(new DefaultOutboundPacket(
+ deviceId,
+ treatment.build(),
+ ByteBuffer.wrap(ethernet.serialize())));
+ }
+
+ private int getPortNum(MacAddress sourceMac, int destinationAddress) {
+ int portNum = findUnusedPortNum();
+ if (portNum == 0) {
+ clearPortNumMap();
+ portNum = findUnusedPortNum();
+ }
+ tpPortNumMap.put(portNum, sourceMac.toString().concat(":").concat(String.valueOf(destinationAddress)));
+ return portNum;
+ }
+
+ private int findUnusedPortNum() {
+ for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
+ if (!tpPortNumMap.containsKey(i)) {
+ return i;
+ }
+ }
+ return 0;
+ }
+
+ private void clearPortNumMap() {
+ tpPortNumMap.entrySet().forEach(e -> {
+ if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
+ tpPortNumMap.remove(e.getKey());
+ }
+ });
+ }
+
+ // TODO there can be multiple routers connected to a particular openstack port
+ // TODO cache router information
+ private Ip4Address getExternalGatewayIp(OpenstackPort osPort) {
+ Optional<OpenstackPort> routerPort = openstackService.ports().stream()
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+ .filter(p -> checkSameSubnet(p, osPort))
+ .findAny();
+ if (!routerPort.isPresent()) {
+ log.warn("No router is connected to network {}", osPort.networkId());
+ return null;
+ }
+
+ OpenstackRouter osRouter = openstackService.router(routerPort.get().deviceId());
+ if (osRouter == null) {
+ log.warn("Failed to get OpenStack router {}",
+ routerPort.get().deviceId());
+ return null;
+ }
+
+ return osRouter.gatewayExternalInfo().externalFixedIps().values()
+ .stream().findAny().orElse(null);
+ }
+
+ private OpenstackPort getOpenstackPort(MacAddress srcMac) {
+ Optional<Host> host = hostService.getHostsByMac(srcMac).stream()
+ .filter(h -> h.annotations().value(PORT_ID) != null)
+ .findAny();
+ if (!host.isPresent()) {
+ log.warn("Failed to find a host with MAC:{}", srcMac);
+ return null;
+ }
+ return openstackService.port(host.get().annotations().value(PORT_ID));
+ }
+
+ private boolean checkSameSubnet(OpenstackPort osPortA, OpenstackPort osPortB) {
+ return osPortA.fixedIps().keySet().stream()
+ .anyMatch(subnetId -> osPortB.fixedIps().keySet().contains(subnetId));
+ }
+
+ private void populatePnatFlowRules(InboundPacket inboundPacket,
+ OpenstackPort osPort,
+ TpPort patPort,
+ Ip4Address externalIp) {
+ long vni = getVni(osPort.networkId());
+ populatePnatIncomingFlowRules(vni, externalIp, patPort, inboundPacket);
+ populatePnatOutgoingFlowRules(vni, externalIp, patPort, inboundPacket);
+ }
+
+ private long getVni(String netId) {
+ // TODO remove this and use host vxlan annotation if applicable
+ return Long.parseLong(openstackService.network(netId).segmentId());
+ }
+
+ private void populatePnatOutgoingFlowRules(long vni, Ip4Address externalIp, TpPort patPort,
+ InboundPacket inboundPacket) {
+ IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(iPacket.getProtocol())
+ .matchTunnelId(vni)
+ .matchIPSrc(IpPrefix.valueOf(iPacket.getSourceAddress(), 32))
+ .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+ switch (iPacket.getProtocol()) {
+ case IPv4.PROTOCOL_TCP:
+ TCP tcpPacket = (TCP) iPacket.getPayload();
+ sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getSourcePort()))
+ .matchTcpDst(TpPort.tpPort(tcpPacket.getDestinationPort()));
+ tBuilder.setTcpSrc(patPort)
+ .setEthDst(DEFAULT_EXTERNAL_ROUTER_MAC);
+ break;
+ case IPv4.PROTOCOL_UDP:
+ UDP udpPacket = (UDP) iPacket.getPayload();
+ sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getSourcePort()))
+ .matchUdpDst(TpPort.tpPort(udpPacket.getDestinationPort()));
+ tBuilder.setUdpSrc(patPort)
+ .setEthDst(DEFAULT_EXTERNAL_ROUTER_MAC);
+
+ break;
+ default:
+ log.debug("Unsupported IPv4 protocol {}");
+ break;
+ }
+
+ tBuilder.setIpSrc(externalIp);
+ gatewayService.getGatewayNodes().stream().forEach(gateway -> {
+ TrafficTreatment.Builder tmpBuilder = tBuilder;
+ tmpBuilder.setOutput(gatewayService.getUplinkPort(gateway.getGatewayDeviceId()));
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tmpBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(PNAT_RULE_PRIORITY)
+ .makeTemporary(PNAT_TIMEOUT)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(gateway.getGatewayDeviceId(), fo);
+ });
+ }
+
+ private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp, TpPort patPort,
+ InboundPacket inboundPacket) {
+ IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
+ IpAddress internalIp = IpAddress.valueOf(iPacket.getSourceAddress());
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(iPacket.getProtocol())
+ .matchIPDst(IpPrefix.valueOf(externalIp, 32))
+ .matchIPSrc(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+ tBuilder.setTunnelId(vni)
+ .setEthDst(inboundPacket.parsed().getSourceMAC())
+ .setIpDst(internalIp);
+
+ switch (iPacket.getProtocol()) {
+ case IPv4.PROTOCOL_TCP:
+ TCP tcpPacket = (TCP) iPacket.getPayload();
+ sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getDestinationPort()))
+ .matchTcpDst(patPort);
+ tBuilder.setTcpDst(TpPort.tpPort(tcpPacket.getSourcePort()));
+ break;
+ case IPv4.PROTOCOL_UDP:
+ UDP udpPacket = (UDP) iPacket.getPayload();
+ sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getDestinationPort()))
+ .matchUdpDst(patPort);
+ tBuilder.setUdpDst(TpPort.tpPort(udpPacket.getSourcePort()));
+ break;
+ default:
+ break;
+ }
+
+ Optional<Host> srcVm = Tools.stream(hostService.getHostsByIp(internalIp))
+ .filter(host -> Objects.equals(
+ host.annotations().value(VXLAN_ID),
+ String.valueOf(vni)))
+ .findFirst();
+ if (!srcVm.isPresent()) {
+ log.warn("Failed to find source VM with IP {}", internalIp);
return;
}
- treatment.setOutput(gatewayService.getUplinkPort(deviceId));
- ethernet.resetChecksum();
+ gatewayService.getGatewayDeviceIds().stream().forEach(deviceId -> {
+ DeviceId srcDeviceId = srcVm.get().location().deviceId();
+ TrafficTreatment.Builder tmpBuilder = tBuilder;
+ tmpBuilder.extension(RulePopulatorUtil.buildExtension(
+ deviceService,
+ deviceId,
+ nodeService.dataIp(srcDeviceId).get().getIp4Address()), deviceId)
+ .setOutput(nodeService.tunnelPort(deviceId).get());
- packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
- ByteBuffer.wrap(ethernet.serialize())));
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tmpBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(PNAT_RULE_PRIORITY)
+ .makeTemporary(PNAT_TIMEOUT)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ });
+ }
+
+ private class InternalPacketProcessor implements PacketProcessor {
+
+ @Override
+ public void process(PacketContext context) {
+ if (context.isHandled()) {
+ return;
+ } else if (!gatewayService.getGatewayDeviceIds().contains(
+ context.inPacket().receivedFrom().deviceId())) {
+ // return if the packet is not from gateway nodes
+ return;
+ }
+
+ InboundPacket pkt = context.inPacket();
+ Ethernet ethernet = pkt.parsed();
+ if (ethernet == null || ethernet.getEtherType() == Ethernet.TYPE_ARP) {
+ return;
+ }
+
+ IPv4 iPacket = (IPv4) ethernet.getPayload();
+ switch (iPacket.getProtocol()) {
+ case IPv4.PROTOCOL_ICMP:
+ break;
+ case IPv4.PROTOCOL_UDP:
+ UDP udpPacket = (UDP) iPacket.getPayload();
+ if (udpPacket.getDestinationPort() == UDP.DHCP_SERVER_PORT &&
+ udpPacket.getSourcePort() == UDP.DHCP_CLIENT_PORT) {
+ // don't process DHCP
+ break;
+ }
+ default:
+ eventExecutor.execute(() -> processPnatPacket(context, ethernet));
+ break;
+ }
+ }
}
}
\ No newline at end of file
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
index 2c9a304..e31f410 100644
--- a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
@@ -15,99 +15,74 @@
*/
package org.onosproject.openstacknetworking.routing;
-import com.google.common.collect.Lists;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.onlab.packet.ARP;
-import org.onlab.packet.EthType;
import org.onlab.packet.Ethernet;
import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
-import org.onosproject.core.ApplicationId;
-import org.onosproject.net.DeviceId;
-import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
-import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.InboundPacket;
import org.onosproject.net.packet.PacketContext;
-import org.onosproject.net.packet.PacketPriority;
+import org.onosproject.net.packet.PacketProcessor;
import org.onosproject.net.packet.PacketService;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.onosproject.openstacknetworking.Constants;
-import org.onosproject.openstacknode.OpenstackNodeService;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Optional;
+import java.util.concurrent.ExecutorService;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.Constants.DEVICE_OWNER_FLOATING_IP;
+import static org.onosproject.openstacknetworking.Constants.DEVICE_OWNER_ROUTER_GATEWAY;
import static org.slf4j.LoggerFactory.getLogger;
/**
- * Handle ARP packet sent from Openstack Gateway nodes.
+ * Handle ARP, ICMP and NAT packets from gateway nodes.
*/
+@Component(immediate = true)
public class OpenstackRoutingArpHandler {
- protected final Logger log = getLogger(getClass());
+ private final Logger log = getLogger(getClass());
- private final PacketService packetService;
- private final OpenstackInterfaceService openstackService;
- private final ScalableGatewayService gatewayService;
- private final OpenstackNodeService nodeService;
- private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
- private static final String NETWORK_FLOATING_IP = "network:floatingip";
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected PacketService packetService;
- /**
- * Default constructor.
- *
- * @param packetService packet service
- * @param openstackService openstackInterface service
- * @param gatewayService gateway service
- * @param nodeService openstackNodeService
- */
- OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
- OpenstackNodeService nodeService, ScalableGatewayService gatewayService) {
- this.packetService = packetService;
- this.openstackService = checkNotNull(openstackService);
- this.nodeService = nodeService;
- this.gatewayService = gatewayService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackInterfaceService openstackService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ScalableGatewayService gatewayService;
+
+ private final ExecutorService executorService =
+ newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "packet-event", log));
+
+ private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
+
+ @Activate
+ protected void activate() {
+ packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
+ log.info("Started");
}
- /**
- * Requests ARP packet to GatewayNode.
- *
- * @param appId application id
- */
- public void requestPacket(ApplicationId appId) {
-
- TrafficSelector arpSelector = DefaultTrafficSelector.builder()
- .matchEthType(EthType.EtherType.ARP.ethType().toShort())
- .build();
-
- getExternalInfo().forEach(deviceId ->
- packetService.requestPackets(arpSelector,
- PacketPriority.CONTROL,
- appId,
- Optional.of(deviceId))
- );
+ @Deactivate
+ protected void deactivate() {
+ packetService.removeProcessor(packetProcessor);
+ log.info("Stopped");
}
- /**
- * Handles ARP packet.
- *
- * @param context packet context
- * @param ethernet ethernet
- */
- public void processArpPacketFromRouter(PacketContext context, Ethernet ethernet) {
- checkNotNull(context, "context can not be null");
- checkNotNull(ethernet, "ethernet can not be null");
-
-
+ private void processArpPacket(PacketContext context, Ethernet ethernet) {
ARP arp = (ARP) ethernet.getPayload();
-
- log.debug("arpEvent called from {} to {}",
+ log.trace("arpEvent called from {} to {}",
Ip4Address.valueOf(arp.getSenderProtocolAddress()).toString(),
Ip4Address.valueOf(arp.getTargetProtocolAddress()).toString());
@@ -116,13 +91,11 @@
}
IpAddress targetIp = Ip4Address.valueOf(arp.getTargetProtocolAddress());
-
if (getTargetMacForTargetIp(targetIp.getIp4Address()) == MacAddress.NONE) {
- return;
+ return;
}
- // FIXME: Set the correct gateway
- MacAddress targetMac = Constants.GW_EXT_INT_MAC;
+ MacAddress targetMac = Constants.DEFAULT_EXTERNAL_ROUTER_MAC;
Ethernet ethReply = ARP.buildArpReply(targetIp.getIp4Address(),
targetMac, ethernet);
@@ -136,22 +109,35 @@
ByteBuffer.wrap(ethReply.serialize())));
}
+ private class InternalPacketProcessor implements PacketProcessor {
+
+ @Override
+ public void process(PacketContext context) {
+ if (context.isHandled()) {
+ return;
+ } else if (!gatewayService.getGatewayDeviceIds().contains(
+ context.inPacket().receivedFrom().deviceId())) {
+ // return if the packet is not from gateway nodes
+ return;
+ }
+
+ InboundPacket pkt = context.inPacket();
+ Ethernet ethernet = pkt.parsed();
+ if (ethernet != null &&
+ ethernet.getEtherType() == Ethernet.TYPE_ARP) {
+ executorService.execute(() -> processArpPacket(context, ethernet));
+ }
+ }
+ }
+
+ // TODO make a cache for the MAC, not a good idea to REST call every time it gets ARP request
private MacAddress getTargetMacForTargetIp(Ip4Address targetIp) {
OpenstackPort port = openstackService.ports().stream()
- .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_GATEWAY) ||
- p.deviceOwner().equals(NETWORK_FLOATING_IP))
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_GATEWAY) ||
+ p.deviceOwner().equals(DEVICE_OWNER_FLOATING_IP))
.filter(p -> p.fixedIps().containsValue(targetIp.getIp4Address()))
.findAny().orElse(null);
- if (port == null) {
- return MacAddress.NONE;
- }
- return port.macAddress();
- }
-
- private List<DeviceId> getExternalInfo() {
- List<DeviceId> externalInfoList = Lists.newArrayList();
- gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
- return externalInfoList;
+ return port == null ? MacAddress.NONE : port.macAddress();
}
}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
index 9f19a28..09b4b8a 100644
--- a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
@@ -15,8 +15,7 @@
*/
package org.onosproject.openstacknetworking.routing;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.ImmutableSet;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
@@ -24,82 +23,62 @@
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.Service;
import org.onlab.packet.Ethernet;
-import org.onlab.packet.IPv4;
import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
-import org.onlab.packet.UDP;
-import org.onlab.util.KryoNamespace;
+import org.onlab.util.Tools;
import org.onosproject.core.ApplicationId;
-import org.onosproject.core.CoreService;
-import org.onosproject.mastership.MastershipService;
-import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.core.GroupId;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Host;
-import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.host.DefaultHostDescription;
-import org.onosproject.net.host.HostDescription;
-import org.onosproject.net.host.HostEvent;
-import org.onosproject.net.host.HostListener;
-import org.onosproject.net.host.HostProvider;
-import org.onosproject.net.host.HostProviderRegistry;
-import org.onosproject.net.host.HostProviderService;
-import org.onosproject.net.host.HostService;
-import org.onosproject.net.packet.InboundPacket;
-import org.onosproject.net.packet.PacketContext;
-import org.onosproject.net.packet.PacketProcessor;
-import org.onosproject.net.packet.PacketService;
-import org.onosproject.net.provider.AbstractProvider;
-import org.onosproject.net.provider.ProviderId;
-import org.onosproject.openstackinterface.OpenstackFloatingIP;
+import org.onosproject.net.flowobjective.ForwardingObjective;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackNetwork;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstackinterface.OpenstackRouterInterface;
-import org.onosproject.openstacknetworking.OpenstackRoutingService;
-import org.onosproject.scalablegateway.api.ScalableGatewayService;
-import org.onosproject.openstacknetworking.routing.OpenstackFloatingIPHandler.Action;
+import org.onosproject.openstackinterface.OpenstackSubnet;
+import org.onosproject.openstacknetworking.AbstractVmHandler;
import org.onosproject.openstacknetworking.Constants;
+import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.onosproject.openstacknetworking.RulePopulatorUtil;
import org.onosproject.openstacknode.OpenstackNode;
import org.onosproject.openstacknode.OpenstackNodeEvent;
import org.onosproject.openstacknode.OpenstackNodeListener;
import org.onosproject.openstacknode.OpenstackNodeService;
-import org.onosproject.store.serializers.KryoNamespaces;
-import org.onosproject.store.service.ConsistentMap;
-import org.onosproject.store.service.Serializer;
-import org.onosproject.store.service.StorageService;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
-import java.util.List;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.stream.Collectors;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static org.onlab.util.Tools.groupedThreads;
-import static org.onosproject.net.AnnotationKeys.PORT_NAME;
+import static org.onosproject.openstacknetworking.Constants.*;
+import static org.onosproject.openstacknetworking.RulePopulatorUtil.buildExtension;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.COMPUTE;
+import static org.onosproject.openstacknode.OpenstackNodeService.NodeType.GATEWAY;
@Component(immediate = true)
@Service
-/**
- * Populates flow rules about L3 functionality for VMs in Openstack.
- */
-public class OpenstackRoutingManager implements OpenstackRoutingService {
+public class OpenstackRoutingManager extends AbstractVmHandler implements OpenstackRoutingService {
private final Logger log = LoggerFactory.getLogger(getClass());
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected CoreService coreService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected PacketService packetService;
+ protected FlowObjectiveService flowObjectiveService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected DeviceService deviceService;
@@ -108,620 +87,392 @@
protected OpenstackInterfaceService openstackService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected FlowObjectiveService flowObjectiveService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected DriverService driverService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected StorageService storageService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected HostService hostService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected HostProviderRegistry hostProviderRegistry;
+ protected OpenstackNodeService nodeService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ScalableGatewayService gatewayService;
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected OpenstackNodeService nodeService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected MastershipService mastershipService;
+ private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+ private final InternalNodeListener nodeListener = new InternalNodeListener();
private ApplicationId appId;
- private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
- private ConsistentMap<String, OpenstackFloatingIP> floatingIpMap; // Map<FloatingIp`s Id, FloatingIp object>
- // Map<RouterInterface`s portId, Corresponded port`s network id>
- private ConsistentMap<String, String> routerInterfaceMap;
- private static final ProviderId PID = new ProviderId("of", "org.onosproject.openstackroutering", true);
- private static final String APP_ID = "org.onosproject.openstackrouting";
- private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
- private static final String FLOATING_IP_MAP_NAME = "openstackrouting-floatingip";
- private static final String TP_PORT_MAP_NAME = "openstackrouting-tpportnum";
- private static final String ROUTER_INTERFACE_MAP_NAME = "openstackrouting-routerinterface";
- private static final String COLON = ":";
- private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
- private static final int TP_PORT_MINIMUM_NUM = 1024;
- private static final int TP_PORT_MAXIMUM_NUM = 65535;
-
- private static final KryoNamespace.Builder FLOATING_IP_SERIALIZER = KryoNamespace.newBuilder()
- .register(KryoNamespaces.API)
- .register(OpenstackFloatingIP.FloatingIpStatus.class)
- .register(OpenstackFloatingIP.class);
-
- private static final KryoNamespace.Builder NUMBER_SERIALIZER = KryoNamespace.newBuilder()
- .register(KryoNamespaces.API);
-
- private static final KryoNamespace.Builder ROUTER_INTERFACE_SERIALIZER = KryoNamespace.newBuilder()
- .register(KryoNamespaces.API);
-
- private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
- private InternalHostListener internalHostListener = new InternalHostListener();
- private InternalOpenstackNodeListener internalNodeListener = new InternalOpenstackNodeListener();
- private ExecutorService l3EventExecutorService =
- Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "L3-event"));
- private ExecutorService icmpEventExecutorService =
- Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "icmp-event"));
- private ExecutorService arpEventExecutorService =
- Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "arp-event"));
- private OpenstackIcmpHandler openstackIcmpHandler;
- private OpenstackRoutingArpHandler openstackArpHandler;
- private OpenstackRoutingRulePopulator rulePopulator;
-
- private HostProviderService hostProviderService;
- private final HostProvider hostProvider = new InternalHostProvider();
@Activate
protected void activate() {
- appId = coreService.registerApplication(APP_ID);
- hostService.addListener(internalHostListener);
- nodeService.addListener(internalNodeListener);
- hostProviderService = hostProviderRegistry.register(hostProvider);
-
- floatingIpMap = storageService.<String, OpenstackFloatingIP>consistentMapBuilder()
- .withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
- .withName(FLOATING_IP_MAP_NAME)
- .withApplicationId(appId)
- .build();
- tpPortNumMap = storageService.<Integer, String>consistentMapBuilder()
- .withSerializer(Serializer.using(NUMBER_SERIALIZER.build()))
- .withName(TP_PORT_MAP_NAME)
- .withApplicationId(appId)
- .build();
- routerInterfaceMap = storageService.<String, String>consistentMapBuilder()
- .withSerializer(Serializer.using(ROUTER_INTERFACE_SERIALIZER.build()))
- .withName(ROUTER_INTERFACE_MAP_NAME)
- .withApplicationId(appId)
- .build();
-
- log.info("started");
+ super.activate();
+ appId = coreService.registerApplication(ROUTING_APP_ID);
+ nodeService.addListener(nodeListener);
}
@Deactivate
protected void deactivate() {
- packetService.removeProcessor(internalPacketProcessor);
- hostService.removeListener(internalHostListener);
- nodeService.removeListener(internalNodeListener);
-
- l3EventExecutorService.shutdown();
- icmpEventExecutorService.shutdown();
- arpEventExecutorService.shutdown();
-
- floatingIpMap.clear();
- tpPortNumMap.clear();
- routerInterfaceMap.clear();
-
+ nodeService.removeListener(nodeListener);
log.info("stopped");
}
-
@Override
- public void createFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
- floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
+ public void createRouter(OpenstackRouter osRouter) {
}
@Override
- public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
- if (!floatingIpMap.containsKey(openstackFloatingIp.id())) {
- log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIp.id());
+ public void updateRouter(OpenstackRouter osRouter) {
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ openstackService.ports().stream()
+ .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ osPort.deviceId().equals(osRouter.id()))
+ .forEach(osPort -> setExternalConnection(osRouter, osPort.networkId()));
+
+ log.info("Connected external gateway {} to router {}",
+ osRouter.gatewayExternalInfo().externalFixedIps(),
+ osRouter.name());
+ } else {
+ openstackService.ports().stream()
+ .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ osPort.deviceId().equals(osRouter.id()))
+ .forEach(osPort -> unsetExternalConnection(osRouter, osPort.networkId()));
+
+ log.info("Disconnected external gateway from router {}",
+ osRouter.name());
+ }
+ }
+
+ @Override
+ public void removeRouter(String osRouterId) {
+ // TODO implement this
+ }
+
+ @Override
+ public void addRouterInterface(OpenstackRouterInterface routerIface) {
+ OpenstackRouter osRouter = openstackRouter(routerIface.id());
+ OpenstackPort osPort = openstackService.port(routerIface.portId());
+ if (osRouter == null || osPort == null) {
+ log.warn("Failed to add router interface {}", routerIface);
return;
}
- if (openstackFloatingIp.portId() == null || openstackFloatingIp.portId().equals("null")) {
- OpenstackFloatingIP floatingIp = floatingIpMap.get(openstackFloatingIp.id()).value();
- // XXX When the VM has been removed, host information has been removed or not ???
- Optional<Host> host = hostService.getHostsByIp(openstackFloatingIp.fixedIpAddress().getIp4Address())
- .stream()
- .findFirst();
- if (!host.isPresent()) {
- log.warn("No Host info with the VM IP the Floating IP address {} is found",
- openstackFloatingIp.floatingIpAddress());
- return;
- }
- l3EventExecutorService.execute(
- new OpenstackFloatingIPHandler(rulePopulator, floatingIp, Action.DISSASSOCIATE, host.get()));
- floatingIpMap.replace(floatingIp.id(), openstackFloatingIp);
- registerFloatingIpToHostService(openstackFloatingIp, Action.DISSASSOCIATE);
- } else {
- floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
- l3EventExecutorService.execute(
- new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIp, Action.ASSOCIATE, null));
- registerFloatingIpToHostService(openstackFloatingIp, Action.ASSOCIATE);
+
+ setRoutes(osRouter, Optional.empty());
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ setExternalConnection(osRouter, osPort.networkId());
}
+ log.info("Connected {} to router {}", osPort.fixedIps(), osRouter.name());
}
@Override
- public void deleteFloatingIP(String id) {
- floatingIpMap.remove(id);
- }
-
- @Override
- public void createRouter(OpenstackRouter openstackRouter) {
- }
-
- @Override
- public void updateRouter(OpenstackRouter openstackRouter) {
- if (openstackRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
- checkExternalConnection(openstackRouter, getOpenstackRouterInterface(openstackRouter));
- } else {
- unsetExternalConnection();
- }
- }
-
- private void unsetExternalConnection() {
- Collection<OpenstackRouter> internalRouters = getExternalRouter(false);
- internalRouters.forEach(r ->
- getOpenstackRouterInterface(r).forEach(i -> rulePopulator.removeExternalRules(i)));
- }
-
- private Collection<OpenstackRouter> getExternalRouter(boolean externalConnection) {
- List<OpenstackRouter> routers;
- if (externalConnection) {
- routers = openstackService.routers()
- .stream()
- .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() > 0))
- .collect(Collectors.toList());
- } else {
- routers = openstackService.routers()
- .stream()
- .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() == 0))
- .collect(Collectors.toList());
- }
- return routers;
- }
-
- @Override
- public void deleteRouter(String id) {
- //TODO : In now, there`s nothing to do for deleteRouter process. It is reserved.
- }
-
- @Override
- public void updateRouterInterface(OpenstackRouterInterface routerInterface) {
- List<OpenstackRouterInterface> routerInterfaces = Lists.newArrayList();
- routerInterfaces.add(routerInterface);
- checkExternalConnection(getOpenstackRouter(routerInterface.id()), routerInterfaces);
- setL3Connection(getOpenstackRouter(routerInterface.id()), null);
- routerInterfaceMap.put(routerInterface.portId(), openstackService.port(routerInterface.portId()).networkId());
- }
-
- /**
- * Set flow rules for traffic between two different subnets when more than one subnets
- * connected to a router.
- *
- * @param openstackRouter OpenstackRouter Info
- * @param openstackPort OpenstackPort Info
- */
- private void setL3Connection(OpenstackRouter openstackRouter, OpenstackPort openstackPort) {
- Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(openstackRouter);
-
- if (interfaceList.size() < 2) {
+ public void removeRouterInterface(OpenstackRouterInterface routerIface) {
+ OpenstackRouter osRouter = openstackService.router(routerIface.id());
+ if (osRouter == null) {
+ log.warn("Failed to remove router interface {}", routerIface);
return;
}
- if (openstackPort == null) {
- interfaceList.forEach(i -> {
- OpenstackPort interfacePort = openstackService.port(i.portId());
- openstackService.ports()
- .stream()
- .filter(p -> p.networkId().equals(interfacePort.networkId())
- && !p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
- .forEach(p -> rulePopulator.populateL3Rules(p,
- getL3ConnectionList(p.networkId(), interfaceList)));
- });
- } else {
- rulePopulator.populateL3Rules(openstackPort, getL3ConnectionList(openstackPort.networkId(), interfaceList));
+ OpenstackSubnet osSubnet = openstackService.subnet(routerIface.subnetId());
+ OpenstackNetwork osNet = openstackService.network(osSubnet.networkId());
+
+ unsetRoutes(osRouter, osNet);
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ unsetExternalConnection(osRouter, osNet.id());
}
-
+ log.info("Disconnected {} from router {}", osSubnet.cidr(), osRouter.name());
}
- private List<OpenstackRouterInterface> getL3ConnectionList(String networkId,
- Collection<OpenstackRouterInterface> interfaceList) {
- List<OpenstackRouterInterface> targetList = Lists.newArrayList();
- interfaceList.forEach(i -> {
- OpenstackPort port = openstackService.port(i.portId());
- if (!port.networkId().equals(networkId)) {
- targetList.add(i);
- }
- });
- return targetList;
- }
-
- @Override
- public void removeRouterInterface(OpenstackRouterInterface routerInterface) {
- OpenstackRouter router = openstackService.router(routerInterface.id());
- Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(router);
- if (interfaceList.size() == 1) {
- List<OpenstackRouterInterface> newList = Lists.newArrayList();
- newList.add(routerInterface);
- interfaceList.forEach(i -> removeL3RulesForRouterInterface(i, router, newList));
- }
- removeL3RulesForRouterInterface(routerInterface, router, null);
- rulePopulator.removeExternalRules(routerInterface);
- routerInterfaceMap.remove(routerInterface.portId());
- }
-
- private void removeL3RulesForRouterInterface(OpenstackRouterInterface routerInterface, OpenstackRouter router,
- List<OpenstackRouterInterface> newList) {
- if (!routerInterfaceMap.containsKey(routerInterface.portId())) {
- log.warn("No router interface information found for {}", routerInterface.portId());
+ private void setExternalConnection(OpenstackRouter osRouter, String osNetId) {
+ if (!osRouter.gatewayExternalInfo().isEnablePnat()) {
+ log.debug("Source NAT is disabled");
return;
}
- openstackService.ports(routerInterfaceMap.get(routerInterface.portId()).value()).forEach(p -> {
- Ip4Address vmIp = (Ip4Address) p.fixedIps().values().toArray()[0];
- if (newList == null) {
- rulePopulator.removeL3Rules(vmIp,
- getL3ConnectionList(p.networkId(), getOpenstackRouterInterface(router)));
- } else {
- rulePopulator.removeL3Rules(vmIp, newList);
- }
- }
- );
+
+ // FIXME router interface is subnet specific, not network
+ OpenstackNetwork osNet = openstackService.network(osNetId);
+ populateExternalRules(osNet);
}
- @Override
- public String networkIdForRouterInterface(String portId) {
- return routerInterfaceMap.get(portId).value();
- }
-
- private Collection<OpenstackFloatingIP> associatedFloatingIps() {
- List<OpenstackFloatingIP> fIps = Lists.newArrayList();
- floatingIpMap.values()
- .stream()
- .filter(fIp -> fIp.value().portId() != null)
- .forEach(fIp -> fIps.add(fIp.value()));
- return fIps;
- }
-
- private void reloadInitL3Rules() {
-
- l3EventExecutorService.execute(() ->
- openstackService.ports()
- .stream()
- .forEach(p ->
- {
- if (p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)) {
- updateRouterInterface(portToRouterInterface(p));
- } else {
- Optional<Ip4Address> vmIp = p.fixedIps().values().stream().findAny();
- if (vmIp.isPresent()) {
- OpenstackFloatingIP floatingIP = getOpenstackFloatingIp(vmIp.get());
- if (floatingIP != null) {
- updateFloatingIP(floatingIP);
- }
- }
- }
- })
- );
- }
-
- private OpenstackRouterInterface portToRouterInterface(OpenstackPort p) {
- OpenstackRouterInterface.Builder osBuilder = new OpenstackRouterInterface.Builder()
- .id(checkNotNull(p.deviceId()))
- .tenantId(checkNotNull(openstackService.network(p.networkId()).tenantId()))
- .subnetId(checkNotNull(p.fixedIps().keySet().stream().findFirst().orElse(null)).toString())
- .portId(checkNotNull(p.id()));
-
- return osBuilder.build();
- }
-
- private class InternalPacketProcessor implements PacketProcessor {
-
- @Override
- public void process(PacketContext context) {
-
- DeviceId senderDeviceId = context.inPacket().receivedFrom().deviceId();
- if (!nodeService.routerBridge(senderDeviceId).isPresent()) {
- log.warn("No router bridge for {} is found.", senderDeviceId);
- return;
- }
- if (context.isHandled()) {
- return;
- } else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
- return;
- }
-
- InboundPacket pkt = context.inPacket();
- Ethernet ethernet = pkt.parsed();
-
- //TODO: Considers IPv6 later.
- if (ethernet == null) {
- return;
- } else if (ethernet.getEtherType() == Ethernet.TYPE_IPV4) {
- IPv4 iPacket = (IPv4) ethernet.getPayload();
- switch (iPacket.getProtocol()) {
- case IPv4.PROTOCOL_ICMP:
-
- icmpEventExecutorService.execute(() ->
- openstackIcmpHandler.processIcmpPacket(context, ethernet));
- break;
- case IPv4.PROTOCOL_UDP:
- // don't process DHCP
- UDP udpPacket = (UDP) iPacket.getPayload();
- if (udpPacket.getDestinationPort() == UDP.DHCP_SERVER_PORT &&
- udpPacket.getSourcePort() == UDP.DHCP_CLIENT_PORT) {
- break;
- }
- default:
- int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
- DeviceId deviceId = pkt.receivedFrom().deviceId();
- Port port = null;
- port = deviceService.getPort(deviceId,
- gatewayService.getUplinkPort(deviceId));
- if (port != null) {
- OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
- Ip4Address.valueOf(iPacket.getSourceAddress()));
- l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
- portNum, openstackPort, port));
-
- } else {
- log.warn("There`s no external interface");
- }
-
- break;
- }
- } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
- arpEventExecutorService.execute(() ->
- openstackArpHandler.processArpPacketFromRouter(context, ethernet));
- }
+ private void unsetExternalConnection(OpenstackRouter osRouter, String osNetId) {
+ if (!osRouter.gatewayExternalInfo().isEnablePnat()) {
+ log.debug("Source NAT is disabled");
+ return;
}
- private int getPortNum(MacAddress sourceMac, int destinationAddress) {
- int portNum = findUnusedPortNum();
- if (portNum == 0) {
- clearPortNumMap();
- portNum = findUnusedPortNum();
- }
- tpPortNumMap.put(portNum, sourceMac.toString().concat(COLON).concat(String.valueOf(destinationAddress)));
- return portNum;
- }
-
- private int findUnusedPortNum() {
- for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
- if (!tpPortNumMap.containsKey(i)) {
- return i;
- }
- }
- return 0;
- }
-
+ // FIXME router interface is subnet specific, not network
+ OpenstackNetwork osNet = openstackService.network(osNetId);
+ removeExternalRules(osNet);
}
- private boolean checkGatewayNode(DeviceId deviceId) {
- return gatewayService.getGatewayDeviceIds().contains(deviceId);
+ private void setRoutes(OpenstackRouter osRouter, Optional<Host> host) {
+ Set<OpenstackNetwork> routableNets = routableNetworks(osRouter.id());
+ if (routableNets.size() < 2) {
+ // no other subnet interface is connected to this router, do nothing
+ return;
+ }
+
+ // FIXME router interface is subnet specific, not network
+ Set<String> routableNetIds = routableNets.stream()
+ .map(OpenstackNetwork::id)
+ .collect(Collectors.toSet());
+
+ Set<Host> hosts = host.isPresent() ? ImmutableSet.of(host.get()) :
+ Tools.stream(hostService.getHosts())
+ .filter(h -> routableNetIds.contains(h.annotations().value(NETWORK_ID)))
+ .collect(Collectors.toSet());
+
+ hosts.stream().forEach(h -> populateRoutingRules(h, routableNets));
}
- private void clearPortNumMap() {
- tpPortNumMap.entrySet().forEach(e -> {
- if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
- tpPortNumMap.remove(e.getKey());
- }
+ private void unsetRoutes(OpenstackRouter osRouter, OpenstackNetwork osNet) {
+ // FIXME router interface is subnet specific, not network
+ Set<OpenstackNetwork> routableNets = routableNetworks(osRouter.id());
+ Tools.stream(hostService.getHosts())
+ .filter(h -> Objects.equals(
+ h.annotations().value(NETWORK_ID), osNet.id()))
+ .forEach(h -> removeRoutingRules(h, routableNets));
+
+ routableNets.stream().forEach(n -> {
+ Tools.stream(hostService.getHosts())
+ .filter(h -> Objects.equals(
+ h.annotations().value(NETWORK_ID),
+ n.id()))
+ .forEach(h -> removeRoutingRules(h, ImmutableSet.of(osNet)));
+ log.debug("Removed between {} to {}", n.name(), osNet.name());
});
}
- private Optional<Port> getExternalPort(DeviceId deviceId, String interfaceName) {
- return deviceService.getPorts(deviceId)
- .stream()
- .filter(p -> p.annotations().value(PORT_NAME).equals(interfaceName))
- .findAny();
- }
-
- private void checkExternalConnection(OpenstackRouter router,
- Collection<OpenstackRouterInterface> interfaces) {
- checkNotNull(router, "Router can not be null");
- checkNotNull(interfaces, "routerInterfaces can not be null");
- Ip4Address externalIp = router.gatewayExternalInfo().externalFixedIps()
- .values().stream().findFirst().orElse(null);
- if ((externalIp == null) || (!router.gatewayExternalInfo().isEnablePnat())) {
- log.debug("Not satisfied to set pnat configuration");
- return;
- }
- interfaces.forEach(this::initiateL3Rule);
- }
-
- private Optional<OpenstackRouter> getRouterfromExternalIp(Ip4Address externalIp) {
- return getExternalRouter(true)
- .stream()
- .filter(r -> r.gatewayExternalInfo()
- .externalFixedIps()
- .values()
- .stream()
- .findAny()
- .get()
- .equals(externalIp))
- .findAny();
- }
-
- private void initiateL3Rule(OpenstackRouterInterface routerInterface) {
- long vni = Long.parseLong(openstackService.network(openstackService
- .port(routerInterface.portId()).networkId()).segmentId());
- rulePopulator.populateExternalRules(vni);
- }
-
- private Collection<OpenstackRouterInterface> getOpenstackRouterInterface(OpenstackRouter router) {
- List<OpenstackRouterInterface> interfaces = Lists.newArrayList();
- openstackService.ports()
- .stream()
- .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
- && p.deviceId().equals(router.id()))
- .forEach(p -> interfaces.add(portToRouterInterface(p)));
- return interfaces;
- }
-
- private OpenstackRouter getOpenstackRouter(String id) {
+ private OpenstackRouter openstackRouter(String routerId) {
return openstackService.routers().stream().filter(r ->
- r.id().equals(id)).iterator().next();
+ r.id().equals(routerId)).iterator().next();
}
- private OpenstackPort getOpenstackPort(MacAddress sourceMac, Ip4Address ip4Address) {
- OpenstackPort openstackPort = openstackService.ports().stream()
- .filter(p -> p.macAddress().equals(sourceMac)).iterator().next();
- return openstackPort.fixedIps().values().stream().filter(ip ->
- ip.equals(ip4Address)).count() > 0 ? openstackPort : null;
- }
-
- private OpenstackFloatingIP getOpenstackFloatingIp(Ip4Address vmIp) {
- Optional<OpenstackFloatingIP> floatingIp = floatingIpMap.asJavaMap().values().stream()
- .filter(f -> f.portId() != null && f.fixedIpAddress().equals(vmIp))
- .findAny();
-
- if (floatingIp.isPresent()) {
- return floatingIp.get();
- }
- log.debug("There is no floating IP information for VM IP {}", vmIp);
-
- return null;
- }
-
- private Optional<OpenstackPort> getRouterInterfacePort(String networkId) {
-
- return openstackService.ports()
- .stream()
- .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
- && p.networkId().equals(networkId))
+ private Optional<OpenstackPort> routerIfacePort(String osNetId) {
+ // FIXME router interface is subnet specific, not network
+ return openstackService.ports().stream()
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ p.networkId().equals(osNetId))
.findAny();
}
- // TODO: Remove the function and the related codes when vRouter is running on different ONOS instance.
- private void registerFloatingIpToHostService(OpenstackFloatingIP openstackFloatingIp, Action action) {
+ private Set<OpenstackNetwork> routableNetworks(String osRouterId) {
+ // FIXME router interface is subnet specific, not network
+ return openstackService.ports().stream()
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE) &&
+ p.deviceId().equals(osRouterId))
+ .map(p -> openstackService.network(p.networkId()))
+ .collect(Collectors.toSet());
+ }
- Optional<Host> hostOptional = hostService.getHostsByIp(openstackFloatingIp.fixedIpAddress())
- .stream()
- .findFirst();
- if (!hostOptional.isPresent()) {
- log.warn("No host with IP {} is registered and cannot add the floating IP. ",
- openstackFloatingIp.floatingIpAddress());
+ private void populateExternalRules(OpenstackNetwork osNet) {
+ populateCnodeToGateway(Long.valueOf(osNet.segmentId()));
+ populateGatewayToController(Long.valueOf(osNet.segmentId()));
+ }
+
+ private void removeExternalRules(OpenstackNetwork osNet) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(Long.valueOf(osNet.segmentId()))
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+
+ nodeService.completeNodes().stream().forEach(node -> {
+ ForwardingObjective.Flag flag = node.type().equals(GATEWAY) ?
+ ForwardingObjective.Flag.VERSATILE :
+ ForwardingObjective.Flag.SPECIFIC;
+
+ RulePopulatorUtil.removeRule(
+ flowObjectiveService,
+ appId,
+ node.intBridge(),
+ sBuilder.build(),
+ flag,
+ ROUTING_RULE_PRIORITY);
+ });
+ }
+
+ private void populateRoutingRules(Host host, Set<OpenstackNetwork> osNets) {
+ String osNetId = host.annotations().value(NETWORK_ID);
+ if (osNetId == null) {
return;
}
- Host host = hostOptional.get();
- Set<IpAddress> ipAddresses = Sets.newHashSet();
- if (action == Action.ASSOCIATE) {
- ipAddresses.add(openstackFloatingIp.floatingIpAddress());
+ DeviceId localDevice = host.location().deviceId();
+ PortNumber localPort = host.location().port();
+ if (!nodeService.dataIp(localDevice).isPresent()) {
+ log.warn("Failed to populate L3 rules");
+ return;
}
- HostDescription hostDescription =
- new DefaultHostDescription(host.mac(), host.vlan(), host.location(), ipAddresses,
- (DefaultAnnotations) host.annotations());
+ // TODO improve pipeline, do we have to install access rules between networks
+ // for every single VMs?
+ osNets.stream().filter(osNet -> !osNet.id().equals(osNetId)).forEach(osNet -> {
+ populateRoutingRulestoSameNode(
+ host.ipAddresses().stream().findFirst().get().getIp4Address(),
+ host.mac(),
+ localPort, localDevice,
+ Long.valueOf(osNet.segmentId()));
- hostProviderService.hostDetected(host.id(), hostDescription, false);
+ nodeService.completeNodes().stream()
+ .filter(node -> node.type().equals(COMPUTE))
+ .filter(node -> !node.intBridge().equals(localDevice))
+ .forEach(node -> populateRoutingRulestoDifferentNode(
+ host.ipAddresses().stream().findFirst().get().getIp4Address(),
+ Long.valueOf(osNet.segmentId()),
+ node.intBridge(),
+ nodeService.dataIp(localDevice).get().getIp4Address()));
+ });
}
- private class InternalHostListener implements HostListener {
-
- private void hostDetected(Host host) {
- String portId = host.annotations().value(Constants.PORT_ID);
- OpenstackPort openstackPort = openstackService.port(portId);
- if (openstackPort == null) {
- log.warn("No OpenstackPort information found from OpenStack for port ID {}", portId);
- return;
- }
-
- Optional<OpenstackPort> routerPort = getRouterInterfacePort(openstackPort.networkId());
- if (routerPort.isPresent()) {
- OpenstackRouterInterface routerInterface = portToRouterInterface(routerPort.get());
- l3EventExecutorService.execute(() ->
- setL3Connection(getOpenstackRouter(routerInterface.id()), openstackPort));
-
- }
+ private void removeRoutingRules(Host host, Set<OpenstackNetwork> osNets) {
+ String osNetId = host.annotations().value(NETWORK_ID);
+ if (osNetId == null) {
+ return;
}
- private void hostRemoved(Host host) {
- String portId = host.annotations().value(Constants.PORT_ID);
- OpenstackPort openstackPort = openstackService.port(portId);
- if (openstackPort == null) {
- log.warn("No OpenstackPort information found from OpenStack for port ID {}", portId);
- return;
- }
+ osNets.stream().filter(osNet -> !osNet.id().equals(osNetId)).forEach(osNet -> {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(host.ipAddresses().stream().findFirst().get().toIpPrefix())
+ .matchTunnelId(Long.valueOf(osNet.segmentId()));
- Optional<OpenstackPort> routerPort = getRouterInterfacePort(openstackPort.networkId());
- if (routerPort.isPresent()) {
- OpenstackRouterInterface routerInterface = portToRouterInterface(routerPort.get());
- IpAddress ipAddress = host.ipAddresses().stream().findFirst().get();
- l3EventExecutorService.execute(() -> rulePopulator.removeL3Rules(ipAddress.getIp4Address(),
- getL3ConnectionList(host.annotations().value(Constants.NETWORK_ID),
- getOpenstackRouterInterface(getOpenstackRouter(routerInterface.id())))));
- }
- }
-
- private boolean isValidHost(Host host) {
- return !host.ipAddresses().isEmpty() &&
- host.annotations().value(Constants.VXLAN_ID) != null &&
- host.annotations().value(Constants.NETWORK_ID) != null &&
- host.annotations().value(Constants.TENANT_ID) != null &&
- host.annotations().value(Constants.PORT_ID) != null;
- }
-
- @Override
- public void event(HostEvent event) {
- Host host = event.subject();
- if (!mastershipService.isLocalMaster(host.location().deviceId())) {
- // do not allow to proceed without mastership
- return;
- }
-
- if (!isValidHost(host)) {
- log.debug("Invalid host event, ignore it {}", host);
- return;
- }
-
- switch (event.type()) {
- case HOST_UPDATED:
- case HOST_ADDED:
- l3EventExecutorService.execute(() -> hostDetected(host));
- break;
- case HOST_REMOVED:
- l3EventExecutorService.execute(() -> hostRemoved(host));
- break;
- default:
- break;
- }
- }
+ nodeService.completeNodes().stream()
+ .filter(node -> node.type().equals(COMPUTE))
+ .forEach(node -> RulePopulatorUtil.removeRule(
+ flowObjectiveService,
+ appId,
+ node.intBridge(),
+ sBuilder.build(),
+ ForwardingObjective.Flag.SPECIFIC,
+ ROUTING_RULE_PRIORITY));
+ });
+ log.debug("Removed routing rule from {} to {}", host, osNets);
}
- private class InternalOpenstackNodeListener implements OpenstackNodeListener {
+ private void populateGatewayToController(long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
- private void nodeComplete() {
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+ tBuilder.setOutput(PortNumber.CONTROLLER);
- rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
- deviceService, driverService, nodeService, gatewayService);
- openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService, hostService,
- openstackService, nodeService, gatewayService);
- openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, nodeService,
- gatewayService);
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
- // Packet handlers must be started AFTER all initialization processes.
- packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
+ gatewayService.getGatewayDeviceIds().stream()
+ .forEach(deviceId -> flowObjectiveService.forward(deviceId, fo));
+ }
- openstackIcmpHandler.requestPacket(appId);
- openstackArpHandler.requestPacket(appId);
+ private void populateCnodeToGateway(long vni) {
+ nodeService.completeNodes().stream()
+ .filter(node -> node.type().equals(COMPUTE))
+ .forEach(node -> populateRuleToGateway(
+ node.intBridge(),
+ gatewayService.getGatewayGroupId(node.intBridge()),
+ vni));
+ }
- openstackService.floatingIps().stream()
- .forEach(f -> floatingIpMap.put(f.id(), f));
+ private void populateRuleToGateway(DeviceId deviceId, GroupId groupId, long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
- reloadInitL3Rules();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+
+ tBuilder.group(groupId);
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void populateRoutingRulestoDifferentNode(Ip4Address vmIp, long vni,
+ DeviceId deviceId, Ip4Address hostIp) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchIPDst(vmIp.toIpPrefix());
+ tBuilder.extension(buildExtension(deviceService, deviceId, hostIp), deviceId)
+ .setOutput(nodeService.tunnelPort(deviceId).get());
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void populateRoutingRulestoSameNode(Ip4Address vmIp, MacAddress vmMac,
+ PortNumber port, DeviceId deviceId, long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(vmIp.toIpPrefix())
+ .matchTunnelId(vni);
+
+ tBuilder.setEthDst(vmMac)
+ .setOutput(port);
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void reloadRoutingRules() {
+ eventExecutor.execute(() -> openstackService.ports().stream()
+ .filter(osPort -> osPort.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+ .forEach(osPort -> {
+ OpenstackRouter osRouter = openstackRouter(osPort.deviceId());
+ setRoutes(osRouter, Optional.empty());
+ if (osRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ setExternalConnection(osRouter, osPort.networkId());
+ }
+ }));
+ }
+
+ @Override
+ protected void hostDetected(Host host) {
+ String osNetId = host.annotations().value(NETWORK_ID);
+ Optional<OpenstackPort> routerIface = routerIfacePort(osNetId);
+ if (!routerIface.isPresent()) {
+ return;
}
+ eventExecutor.execute(() -> setRoutes(
+ openstackRouter(routerIface.get().deviceId()),
+ Optional.of(host)));
+ }
+
+ @Override
+ protected void hostRemoved(Host host) {
+ String osNetId = host.annotations().value(NETWORK_ID);
+ Optional<OpenstackPort> routerIface = routerIfacePort(osNetId);
+ if (!routerIface.isPresent()) {
+ return;
+ }
+ Set<OpenstackNetwork> routableNets = routableNetworks(routerIface.get().deviceId());
+ eventExecutor.execute(() -> removeRoutingRules(host, routableNets));
+ }
+
+ private class InternalNodeListener implements OpenstackNodeListener {
@Override
public void event(OpenstackNodeEvent event) {
@@ -730,28 +481,22 @@
switch (event.type()) {
case COMPLETE:
log.info("COMPLETE node {} detected", node.hostname());
- l3EventExecutorService.execute(() -> nodeComplete());
+ if (node.type() == GATEWAY) {
+ GatewayNode gnode = GatewayNode.builder()
+ .gatewayDeviceId(node.intBridge())
+ .dataIpAddress(node.dataIp().getIp4Address())
+ .uplinkIntf(node.externalPortName().get())
+ .build();
+ gatewayService.addGatewayNode(gnode);
+ }
+ eventExecutor.execute(OpenstackRoutingManager.this::reloadRoutingRules);
break;
+ case INIT:
+ case DEVICE_CREATED:
case INCOMPLETE:
- break;
default:
break;
}
}
}
-
- private class InternalHostProvider extends AbstractProvider implements HostProvider {
-
- /**
- * Creates a provider with the supplier identifier.
- */
- protected InternalHostProvider() {
- super(PID);
- }
-
- @Override
- public void triggerProbe(Host host) {
- // nothing to do
- }
- }
}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
deleted file mode 100644
index 875cf98..0000000
--- a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
+++ /dev/null
@@ -1,693 +0,0 @@
-/*
- * Copyright 2016-present Open Networking Laboratory
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.onosproject.openstacknetworking.routing;
-
-import com.google.common.collect.Lists;
-import org.onlab.packet.Ethernet;
-import org.onlab.packet.IPv4;
-import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
-import org.onlab.packet.IpPrefix;
-import org.onlab.packet.MacAddress;
-import org.onlab.packet.TCP;
-import org.onlab.packet.TpPort;
-import org.onlab.packet.UDP;
-import org.onosproject.core.ApplicationId;
-import org.onosproject.core.GroupId;
-import org.onosproject.net.Device;
-import org.onosproject.net.DeviceId;
-import org.onosproject.net.Host;
-import org.onosproject.net.Port;
-import org.onosproject.net.PortNumber;
-import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
-import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.driver.DefaultDriverData;
-import org.onosproject.net.driver.DefaultDriverHandler;
-import org.onosproject.net.driver.Driver;
-import org.onosproject.net.driver.DriverHandler;
-import org.onosproject.net.driver.DriverService;
-import org.onosproject.net.flow.DefaultTrafficSelector;
-import org.onosproject.net.flow.DefaultTrafficTreatment;
-import org.onosproject.net.flow.TrafficSelector;
-import org.onosproject.net.flow.TrafficTreatment;
-import org.onosproject.net.flow.instructions.ExtensionPropertyException;
-import org.onosproject.net.flow.instructions.ExtensionTreatment;
-import org.onosproject.net.flow.instructions.ExtensionTreatmentType;
-import org.onosproject.net.flowobjective.DefaultForwardingObjective;
-import org.onosproject.net.flowobjective.FlowObjectiveService;
-import org.onosproject.net.flowobjective.ForwardingObjective;
-import org.onosproject.net.packet.InboundPacket;
-import org.onosproject.openstackinterface.OpenstackInterfaceService;
-import org.onosproject.openstackinterface.OpenstackPort;
-import org.onosproject.openstackinterface.OpenstackRouterInterface;
-import org.onosproject.openstackinterface.OpenstackSubnet;
-import org.onosproject.openstackinterface.OpenstackFloatingIP;
-import org.onosproject.openstacknetworking.Constants;
-import org.onosproject.openstacknetworking.OpenstackRoutingService;
-import org.onosproject.scalablegateway.api.ScalableGatewayService;
-import org.onosproject.openstacknode.OpenstackNode;
-import org.onosproject.openstacknode.OpenstackNodeService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.StreamSupport;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.onlab.osgi.DefaultServiceDirectory.getService;
-import static org.onosproject.net.AnnotationKeys.PORT_NAME;
-
-/**
- * Populates Routing Flow Rules.
- */
-public class OpenstackRoutingRulePopulator {
-
- private final Logger log = LoggerFactory.getLogger(getClass());
-
- private final ApplicationId appId;
- private final FlowObjectiveService flowObjectiveService;
- private final OpenstackInterfaceService openstackService;
- private final DeviceService deviceService;
- private final DriverService driverService;
- private final ScalableGatewayService gatewayService;
- private final OpenstackNodeService nodeService;
-
- private static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
- private static final String PORTNAME_PREFIX_VM = "tap";
-
- private static final String PORTNOTNULL = "Port can not be null";
- private static final String DEVICENOTNULL = "Device can not be null";
- private static final String TUNNEL_DESTINATION = "tunnelDst";
- private static final int ROUTING_RULE_PRIORITY = 25000;
- private static final int FLOATING_RULE_PRIORITY = 42000;
- private static final int PNAT_RULE_PRIORITY = 26000;
- private static final int PNAT_TIMEOUT = 120;
- private static final int PREFIX_LENGTH = 32;
-
- private InboundPacket inboundPacket;
- private OpenstackPort openstackPort;
- private int portNum;
- private MacAddress externalInterface;
- private MacAddress externalRouter;
-
- /**
- * The constructor of openstackRoutingRulePopulator.
- *
- * @param appId Caller`s appId
- * @param openstackService Opestack REST request handler
- * @param flowObjectiveService FlowObjectiveService
- * @param deviceService DeviceService
- * @param driverService DriverService
- * @param nodeService openstack node service
- * @param gatewayService scalable gateway service
- */
- public OpenstackRoutingRulePopulator(ApplicationId appId,
- OpenstackInterfaceService openstackService,
- FlowObjectiveService flowObjectiveService,
- DeviceService deviceService,
- DriverService driverService,
- OpenstackNodeService nodeService,
- ScalableGatewayService gatewayService) {
- this.appId = appId;
- this.flowObjectiveService = flowObjectiveService;
- this.openstackService = checkNotNull(openstackService);
- this.deviceService = deviceService;
- this.driverService = driverService;
- this.gatewayService = gatewayService;
- this.nodeService = nodeService;
- }
-
- /**
- * Populates flow rules for Pnat configurations.
- *
- * @param inboundPacket Packet-in event packet
- * @param openstackPort Target VM information
- * @param portNum Pnat port number
- * @param externalIp external ip address
- * @param externalInterfaceMacAddress Gateway external interface macaddress
- * @param externalRouterMacAddress Outer(physical) router`s macaddress
- */
- public void populatePnatFlowRules(InboundPacket inboundPacket, OpenstackPort openstackPort, int portNum,
- Ip4Address externalIp, MacAddress externalInterfaceMacAddress,
- MacAddress externalRouterMacAddress) {
- this.inboundPacket = inboundPacket;
- this.openstackPort = openstackPort;
- this.portNum = portNum;
- this.externalInterface = externalInterfaceMacAddress;
- this.externalRouter = externalRouterMacAddress;
-
- long vni = getVni(openstackPort.networkId());
-
- populatePnatIncomingFlowRules(vni, externalIp);
- populatePnatOutgoingFlowRules(vni, externalIp);
- }
-
- private void populatePnatOutgoingFlowRules(long vni, Ip4Address externalIp) {
- IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPProtocol(iPacket.getProtocol())
- .matchTunnelId(vni)
- .matchIPSrc(IpPrefix.valueOf(iPacket.getSourceAddress(), 32))
- .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- switch (iPacket.getProtocol()) {
- case IPv4.PROTOCOL_TCP:
- TCP tcpPacket = (TCP) iPacket.getPayload();
- sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getSourcePort()))
- .matchTcpDst(TpPort.tpPort(tcpPacket.getDestinationPort()));
- tBuilder.setTcpSrc(TpPort.tpPort(portNum));
- break;
- case IPv4.PROTOCOL_UDP:
- UDP udpPacket = (UDP) iPacket.getPayload();
- sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getSourcePort()))
- .matchUdpDst(TpPort.tpPort(udpPacket.getDestinationPort()));
- tBuilder.setUdpSrc(TpPort.tpPort(portNum));
- break;
- default:
- log.debug("Unsupported IPv4 protocol {}");
- break;
- }
-
- tBuilder.setIpSrc(externalIp);
- gatewayService.getGatewayNodes().forEach(node -> {
- tBuilder.setOutput(gatewayService.getUplinkPort(node.getGatewayDeviceId()));
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(PNAT_RULE_PRIORITY)
- .makeTemporary(PNAT_TIMEOUT)
- .fromApp(appId)
- .add();
-
- flowObjectiveService.forward(node.getGatewayDeviceId(), fo);
- });
-
- }
-
- private Port getPortOfExternalInterface() {
- return deviceService.getPorts(getGatewayNode().id()).stream()
- .filter(p -> p.annotations().value(PORT_NAME)
- .equals(org.onosproject.openstacknode.Constants.PATCH_INTG_BRIDGE))
- .findAny().orElse(null);
- }
-
-
- private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp) {
- IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPProtocol(iPacket.getProtocol())
- .matchIPDst(IpPrefix.valueOf(externalIp, 32))
- .matchIPSrc(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
- tBuilder.setTunnelId(vni)
- .setEthDst(inboundPacket.parsed().getSourceMAC())
- .setIpDst(IpAddress.valueOf(iPacket.getSourceAddress()));
-
- switch (iPacket.getProtocol()) {
- case IPv4.PROTOCOL_TCP:
- TCP tcpPacket = (TCP) iPacket.getPayload();
- sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getDestinationPort()))
- .matchTcpDst(TpPort.tpPort(portNum));
- tBuilder.setTcpDst(TpPort.tpPort(tcpPacket.getSourcePort()));
- break;
- case IPv4.PROTOCOL_UDP:
- UDP udpPacket = (UDP) iPacket.getPayload();
- sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getDestinationPort()))
- .matchUdpDst(TpPort.tpPort(portNum));
- tBuilder.setUdpDst(TpPort.tpPort(udpPacket.getSourcePort()));
- break;
- default:
- break;
- }
-
- getGatewayNodeList().forEach(node -> {
- DeviceId deviceId = node.id();
- tBuilder.extension(buildNiciraExtenstion(deviceId,
- getHostIpfromOpenstackPort(openstackPort).getIp4Address()), deviceId)
- .setOutput(getTunnelPort(deviceId));
-
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(PNAT_RULE_PRIORITY)
- .makeTemporary(PNAT_TIMEOUT)
- .fromApp(appId)
- .add();
-
- flowObjectiveService.forward(deviceId, fo);
- });
- }
-
- private List<Device> getGatewayNodeList() {
- List<Device> devices = Lists.newArrayList();
- gatewayService.getGatewayDeviceIds().forEach(deviceId ->
- devices.add(checkNotNull(deviceService.getDevice(deviceId))));
- return devices;
- }
-
- private IpAddress getHostIpfromOpenstackPort(OpenstackPort openstackPort) {
- Device device = getDevicefromOpenstackPort(openstackPort);
-
- Optional<IpAddress> ipAddress = nodeService.dataIp(device.id());
- if (!ipAddress.isPresent()) {
- log.warn("No IP address found for device {}", device.id());
- return null;
- }
-
- return ipAddress.get();
- }
-
- private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) {
- String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
- Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
- .filter(d -> findPortinDevice(d.id(), openstackPortName))
- .iterator()
- .next();
- checkNotNull(device, DEVICENOTNULL);
- return device;
- }
-
- private boolean findPortinDevice(DeviceId deviceId, String openstackPortName) {
- Port port = deviceService.getPorts(deviceId)
- .stream()
- .filter(p -> p.isEnabled() && p.annotations().value(PORT_NAME).equals(openstackPortName))
- .findAny()
- .orElse(null);
- return port != null;
- }
-
- /**
- * Builds Nicira extension for tagging remoteIp of vxlan.
- *
- * @param deviceId Device Id of vxlan source device
- * @param hostIp Remote Ip of vxlan destination device
- * @return NiciraExtension Treatment
- */
- public ExtensionTreatment buildNiciraExtenstion(DeviceId deviceId, Ip4Address hostIp) {
- Driver driver = driverService.getDriver(deviceId);
- DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, deviceId));
- ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
-
- ExtensionTreatment extensionInstruction =
- resolver.getExtensionInstruction(
- ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST.type());
-
- try {
- extensionInstruction.setPropertyValue(TUNNEL_DESTINATION, hostIp);
- } catch (ExtensionPropertyException e) {
- log.error("Error setting Nicira extension setting {}", e);
- }
-
- return extensionInstruction;
- }
-
- /**
- * Returns port number of vxlan tunnel.
- *
- * @param deviceId Target Device Id
- * @return PortNumber
- */
- public PortNumber getTunnelPort(DeviceId deviceId) {
- Port port = deviceService.getPorts(deviceId).stream()
- .filter(p -> p.annotations().value(PORT_NAME).equals(PORTNAME_PREFIX_TUNNEL))
- .findAny().orElse(null);
-
- if (port == null) {
- log.error("No TunnelPort was created.");
- return null;
- }
- return port.number();
-
- }
-
- /**
- * Populates flow rules from openstackComputeNode to GatewayNode.
- *
- * @param vni Target network
- */
- public void populateExternalRules(long vni) {
-
- // 1. computeNode to gateway
- populateComputeNodeRules(vni);
- // 2. gatewayNode to controller
- populateRuleGatewaytoController(vni);
- }
-
- private void populateRuleGatewaytoController(long vni) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchTunnelId(vni)
- .matchEthDst(Constants.GATEWAY_MAC);
- tBuilder.setOutput(PortNumber.CONTROLLER);
-
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(ROUTING_RULE_PRIORITY)
- .fromApp(appId)
- .add();
-
- getGatewayNodeList().forEach(device -> flowObjectiveService.forward(device.id(), fo));
- }
-
- private void populateComputeNodeRules(long vni) {
- StreamSupport.stream(deviceService.getDevices().spliterator(), false)
- .filter(d -> isTypeOf(d.id(), OpenstackNodeService.NodeType.COMPUTE))
- .forEach(d -> populateRuleToGatewayBySgw(d.id(),
- gatewayService.getGatewayGroupId(d.id()), vni));
- }
-
- private void populateRuleToGatewayBySgw(DeviceId deviceId, GroupId groupId, long vni) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchTunnelId(vni)
- .matchEthDst(Constants.GATEWAY_MAC);
-
- tBuilder.group(groupId);
-
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.SPECIFIC)
- .withPriority(ROUTING_RULE_PRIORITY)
- .fromApp(appId)
- .add();
-
- flowObjectiveService.forward(deviceId, fo);
- }
-
- private Device getGatewayNode() {
-
- // TODO Return the correct gateway node
- Optional<OpenstackNode> gwNode = nodeService.nodes().stream()
- .filter(n -> n.type().equals(OpenstackNodeService.NodeType.GATEWAY))
- .findFirst();
-
- if (!gwNode.isPresent()) {
- log.warn("No Gateway is defined.");
- return null;
- }
-
- return deviceService.getDevice(gwNode.get().intBridge());
- }
-
- private boolean isTypeOf(DeviceId deviceId, OpenstackNodeService.NodeType type) {
-
- Optional<OpenstackNode> node = nodeService.nodes().stream()
- .filter(n -> n.intBridge().equals(deviceId) ||
- (n.routerBridge().isPresent() && n.routerBridge().get().equals(deviceId)))
- .filter(n -> n.type().equals(type))
- .findFirst();
-
- if (node.isPresent()) {
- return true;
- }
-
- return false;
- }
-
- private long getVni(String netId) {
- return Long.parseLong(openstackService.network(netId).segmentId());
- }
-
- /**
- * Remove flow rules for external connection.
- *
- * @param routerInterface Corresponding routerInterface
- */
- public void removeExternalRules(OpenstackRouterInterface routerInterface) {
- OpenstackSubnet openstackSubnet = openstackService.subnet(routerInterface.subnetId());
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchTunnelId(getVni(openstackSubnet.networkId()))
- .matchEthDst(Constants.GATEWAY_MAC);
-
- StreamSupport.stream(deviceService.getDevices().spliterator(), false)
- .forEach(d -> {
- ForwardingObjective.Flag flag = isTypeOf(d.id(), OpenstackNodeService.NodeType.GATEWAY) ?
- ForwardingObjective.Flag.VERSATILE :
- ForwardingObjective.Flag.SPECIFIC;
- removeRule(d.id(), sBuilder, flag, ROUTING_RULE_PRIORITY);
- });
-
- }
-
- private void removeRule(DeviceId deviceId, TrafficSelector.Builder sBuilder,
- ForwardingObjective.Flag flag, int priority) {
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(flag)
- .withPriority(priority)
- .fromApp(appId)
- .remove();
-
- flowObjectiveService.forward(deviceId, fo);
- }
-
- /**
- * Populates flow rules for floatingIp configuration.
- *
- * @param floatingIP Corresponding floating ip information
- */
- public void populateFloatingIpRules(OpenstackFloatingIP floatingIP) {
- OpenstackPort port = openstackService.port(floatingIP.portId());
- //1. incoming rules
- populateFloatingIpIncomingRules(floatingIP, port);
- //2. outgoing rules
- populateFloatingIpOutgoingRules(floatingIP, port);
- }
-
- private void populateFloatingIpIncomingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
-
- DeviceId gatewayDeviceId = DeviceId.deviceId(port.deviceId());
- Optional<IpAddress> ipAddress = nodeService.dataIp(gatewayDeviceId);
- if (!ipAddress.isPresent()) {
- log.warn("No IP address found for device {}", port.deviceId());
- return;
- }
- tBuilder.setEthSrc(Constants.GATEWAY_MAC)
- .setEthDst(port.macAddress())
- .setIpDst(floatingIP.fixedIpAddress())
- .setTunnelId(getVni(port.networkId()))
- .extension(buildNiciraExtenstion(gatewayDeviceId,
- ipAddress.get().getIp4Address()), gatewayDeviceId)
- .setOutput(getTunnelPort(gatewayDeviceId));
-
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(FLOATING_RULE_PRIORITY)
- .fromApp(appId)
- .add();
-
- flowObjectiveService.forward(getGatewayNode().id(), fo);
- }
-
- private void populateFloatingIpOutgoingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchTunnelId(getVni(port.networkId()))
- .matchIPSrc(IpPrefix.valueOf(floatingIP.fixedIpAddress(), 32));
-
- getGatewayNodeList().forEach(device -> {
- DeviceId deviceId = device.id();
- tBuilder.setIpSrc(floatingIP.floatingIpAddress())
- .setEthSrc(Constants.GW_EXT_INT_MAC)
- .setEthDst(Constants.PHY_ROUTER_MAC)
- .setOutput(getExternalPortNum(deviceId));
-
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(FLOATING_RULE_PRIORITY)
- .fromApp(appId)
- .add();
-
- flowObjectiveService.forward(deviceId, fo);
- });
- }
-
- private PortNumber getExternalPortNum(DeviceId deviceId) {
- return checkNotNull(gatewayService.getUplinkPort(deviceId), PORTNOTNULL);
- }
-
- /**
- * Removes flow rules for floating ip configuration.
- *
- * @param floatingIp Corresponding floating ip information
- * @param host host information for vm to remove
- */
- public void removeFloatingIpRules(OpenstackFloatingIP floatingIp, Host host) {
- TrafficSelector.Builder sOutgoingBuilder = DefaultTrafficSelector.builder();
- TrafficSelector.Builder sIncomingBuilder = DefaultTrafficSelector.builder();
-
- // XXX FloatingIp.tenant_id() == host.vxlan_id ???
- sOutgoingBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchTunnelId(Integer.parseInt(host.annotations().value(Constants.VXLAN_ID)))
- .matchIPSrc(IpPrefix.valueOf(floatingIp.fixedIpAddress(), PREFIX_LENGTH));
-
- sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(floatingIp.floatingIpAddress(), PREFIX_LENGTH));
-
- getGatewayNodeList().forEach(device -> {
- removeRule(device.id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
- removeRule(device.id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
- });
- }
-
- /**
- * Populates L3 rules for east to west traffic.
- *
- * @param openstackPort target VM
- * @param targetList target openstackRouterInterfaces
- */
- public void populateL3Rules(OpenstackPort openstackPort, List<OpenstackRouterInterface> targetList) {
- Device device = getDevicefromOpenstackPort(openstackPort);
- Port port = getPortFromOpenstackPort(device, openstackPort);
- Ip4Address vmIp = openstackPort.fixedIps().values().iterator().next();
-
- if (port == null) {
- return;
- }
-
- targetList.forEach(routerInterface -> {
- long vni = getVni(openstackService.port(routerInterface.portId()).networkId());
-
- if (vmIp == null) {
- return;
- }
-
- populateL3RulestoSameNode(vmIp, openstackPort, port, device, vni);
-
- deviceService.getAvailableDevices().forEach(d -> {
- if (!d.equals(device) && !d.equals(getGatewayNode())) {
- populateL3RulestoDifferentNode(vmIp, vni, d.id(),
- getHostIpfromOpenstackPort(openstackPort).getIp4Address());
- }
- });
- });
- }
-
- private void populateL3RulestoDifferentNode(Ip4Address vmIp, long vni, DeviceId deviceId, Ip4Address hostIp) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchTunnelId(vni)
- .matchIPDst(vmIp.toIpPrefix());
- tBuilder.extension(buildNiciraExtenstion(deviceId, hostIp), deviceId)
- .setOutput(getTunnelPort(deviceId));
-
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withPriority(ROUTING_RULE_PRIORITY)
- .withFlag(ForwardingObjective.Flag.SPECIFIC)
- .fromApp(appId)
- .add();
-
- flowObjectiveService.forward(deviceId, fo);
- }
-
- private void populateL3RulestoSameNode(Ip4Address vmIp, OpenstackPort p, Port port, Device device, long vni) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(vmIp.toIpPrefix())
- .matchTunnelId(vni);
-
- tBuilder.setEthDst(p.macAddress())
- .setOutput(port.number());
-
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withPriority(ROUTING_RULE_PRIORITY)
- .withFlag(ForwardingObjective.Flag.SPECIFIC)
- .fromApp(appId)
- .add();
-
- flowObjectiveService.forward(device.id(), fo);
- }
-
- private Port getPortFromOpenstackPort(Device device, OpenstackPort p) {
- String openstackPortName = PORTNAME_PREFIX_VM + p.id().substring(0, 11);
- return deviceService.getPorts(device.id())
- .stream()
- .filter(pt -> pt.annotations().value(PORT_NAME).equals(openstackPortName))
- .findAny()
- .orElse(null);
- }
-
- /**
- * Removes L3 rules for routerInterface events.
- *
- * @param vmIp Corresponding Vm ip
- * @param routerInterfaces Corresponding routerInterfaces
- */
- public void removeL3Rules(Ip4Address vmIp, List<OpenstackRouterInterface> routerInterfaces) {
- if (vmIp == null) {
- return;
- }
-
- OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
-
- deviceService.getAvailableDevices().forEach(d -> {
- if (isTypeOf(d.id(), OpenstackNodeService.NodeType.COMPUTE)) {
- routerInterfaces.forEach(routerInterface -> {
- String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
- long vni = getVni(networkId);
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-
- sBuilder.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(vmIp.toIpPrefix())
- .matchTunnelId(vni);
-
- removeRule(d.id(), sBuilder, ForwardingObjective.Flag.SPECIFIC, ROUTING_RULE_PRIORITY);
- });
- }
- });
- }
-}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java
index 78d05a9..67fd1a5 100644
--- a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java
@@ -15,6 +15,8 @@
*/
/**
- * Application for OpenstackRouting.
+ * Implements OpenStack L3 service plugin, which routes packets between subnets,
+ * forwards packets from internal networks to external ones, and accesses instances
+ * from external networks through floating IPs.
*/
-package org.onosproject.openstacknetworking.routing;
\ No newline at end of file
+package org.onosproject.openstacknetworking.routing;