[ONOS-4481] Integrate scalable gateway to openstackRouting
- Modified openstack_routing to use scalable gateway application
- Add GatewayNode Builder method
Change-Id: Ib12d52009eeb89ab59f01cafe794894aac80985e
diff --git a/apps/openstacknetworking/openstackrouting/BUCK b/apps/openstacknetworking/openstackrouting/BUCK
index b691ff9..da4f812 100644
--- a/apps/openstacknetworking/openstackrouting/BUCK
+++ b/apps/openstacknetworking/openstackrouting/BUCK
@@ -3,6 +3,7 @@
'//core/store/serializers:onos-core-serializers',
'//apps/openstackinterface/api:onos-apps-openstackinterface-api',
'//apps/openstacknetworking/api:onos-apps-openstacknetworking-api',
+ '//apps/scalablegateway:onos-apps-scalablegateway',
]
BUNDLES = [
diff --git a/apps/openstacknetworking/openstackrouting/pom.xml b/apps/openstacknetworking/openstackrouting/pom.xml
index 4c6f638..3b74025 100644
--- a/apps/openstacknetworking/openstackrouting/pom.xml
+++ b/apps/openstacknetworking/openstackrouting/pom.xml
@@ -52,6 +52,11 @@
<artifactId>onos-app-openstackinterface-api</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.onosproject</groupId>
+ <artifactId>onos-scalablegateway</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
index 71fb81c..9678598 100644
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
@@ -40,6 +40,7 @@
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackSwitchingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
@@ -62,13 +63,14 @@
private final Map<String, OpenstackPortInfo> icmpInfoMap = Maps.newHashMap();
private final OpenstackSwitchingService openstackSwitchingService;
private final OpenstackInterfaceService openstackService;
+ private final ScalableGatewayService gatewayService;
private final OpenstackNetworkingConfig config;
private static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
private static final String PORTNAME = "portName";
private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
private static final String NETWORK_FLOATING_IP = "network:floatingip";
-
+ private static final String EXTERNAL_NODE_NULL = "There is no external node about this deviceId []";
/**
* Default constructor.
*
@@ -77,15 +79,17 @@
* @param openstackService openstackInterface service
* @param config openstackRoutingConfig
* @param openstackSwitchingService openstackSwitching service
+ * @param gatewayService scalable gateway service
*/
OpenstackIcmpHandler(PacketService packetService, DeviceService deviceService,
OpenstackInterfaceService openstackService, OpenstackNetworkingConfig config,
- OpenstackSwitchingService openstackSwitchingService) {
+ OpenstackSwitchingService openstackSwitchingService, ScalableGatewayService gatewayService) {
this.packetService = packetService;
this.deviceService = deviceService;
this.openstackService = checkNotNull(openstackService);
this.config = checkNotNull(config);
this.openstackSwitchingService = checkNotNull(openstackSwitchingService);
+ this.gatewayService = gatewayService;
}
/**
@@ -99,10 +103,13 @@
.matchIPProtocol(IPv4.PROTOCOL_ICMP)
.build();
- packetService.requestPackets(icmpSelector,
- PacketPriority.CONTROL,
- appId,
- Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
+ Map<DeviceId, PortNumber> externalInfoMap = getExternalInfo();
+
+ externalInfoMap.keySet().forEach(deviceId ->
+ packetService.requestPackets(icmpSelector,
+ PacketPriority.CONTROL,
+ appId,
+ Optional.of(deviceId)));
}
/**
@@ -223,14 +230,19 @@
Ethernet icmpRequestEth = new Ethernet();
icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
- .setSourceMACAddress(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
- .setDestinationMACAddress(MacAddress.valueOf(config.physicalRouterMac()))
.setPayload(icmpRequestIpv4);
- TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- .setOutput(getPortForAnnotationPortName(DeviceId.deviceId(config.gatewayBridgeId()),
- config.gatewayExternalInterfaceName()))
- .build();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ Map<DeviceId, PortNumber> externalInforMap = getExternalInfo();
+
+ if (externalInforMap.size() == 0 || !externalInforMap.containsKey(deviceId)) {
+ log.error(EXTERNAL_NODE_NULL, deviceId.toString());
+ return;
+ }
+ tBuilder.setOutput(externalInforMap.get(deviceId));
+
+ TrafficTreatment treatment = tBuilder.build();
OutboundPacket packet = new DefaultOutboundPacket(deviceId,
treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
@@ -323,4 +335,10 @@
}
return true;
}
-}
\ No newline at end of file
+ private Map<DeviceId, PortNumber> getExternalInfo() {
+ Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
+ gatewayService.getGatewayDeviceIds().forEach(deviceId ->
+ externalInfoMap.putIfAbsent(deviceId, gatewayService.getGatewayExternalPorts(deviceId).get(0)));
+ return externalInfoMap;
+ }
+}
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
index f144e88..3f11e6c 100644
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
@@ -30,10 +30,11 @@
import org.onosproject.net.packet.PacketContext;
import org.onosproject.net.packet.PacketService;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
-import org.onosproject.openstackinterface.OpenstackNetwork;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,6 +61,7 @@
private OpenstackNetworkingConfig config;
private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
+ private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
int portNum, OpenstackPort openstackPort, Port port, OpenstackNetworkingConfig config) {
@@ -84,16 +86,17 @@
OpenstackRouter router = getOpenstackRouter(openstackPort);
+ MacAddress externalMac = MacAddress.NONE;
+ MacAddress routerMac = MacAddress.NONE;
+
rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
- getExternalIp(router), MacAddress.valueOf(config.gatewayExternalInterfaceMac()),
- MacAddress.valueOf(config.physicalRouterMac()));
+ getExternalIp(router), externalMac, routerMac);
packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
}
private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
- OpenstackNetwork network = networkingService.network(openstackPort.networkId());
OpenstackPort port = networkingService.ports()
.stream()
@@ -146,11 +149,16 @@
iPacket.resetChecksum();
iPacket.setParent(ethernet);
ethernet.setPayload(iPacket);
- ethernet.setSourceMACAddress(config.gatewayExternalInterfaceMac())
- .setDestinationMACAddress(config.physicalRouterMac());
+ ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
+ GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
+ if (gatewayNode.getGatewayExternalInterfaceNames().size() == 0) {
+ log.error(EXTERNAL_PORT_NULL, deviceId.toString());
+ return;
+ }
+ treatment.setOutput(gatewayService.getGatewayExternalPorts(deviceId).get(0));
+
ethernet.resetChecksum();
- treatment.setOutput(port.number());
packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
ByteBuffer.wrap(ethernet.serialize())));
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
index 0ad2101..9e23b5d 100644
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
@@ -15,6 +15,7 @@
*/
package org.onosproject.openstacknetworking.routing;
+import com.google.common.collect.Lists;
import org.onlab.packet.ARP;
import org.onlab.packet.EthType;
import org.onlab.packet.Ethernet;
@@ -34,9 +35,11 @@
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
+import java.util.List;
import java.util.Optional;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -51,21 +54,23 @@
private final PacketService packetService;
private final OpenstackInterfaceService openstackService;
private final OpenstackNetworkingConfig config;
+ private final ScalableGatewayService gatewayService;
private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
private static final String NETWORK_FLOATING_IP = "network:floatingip";
/**
* Default constructor.
- *
- * @param packetService packet service
+ * @param packetService packet service
* @param openstackService openstackInterface service
* @param config openstackRoutingConfig
+ * @param gatewayService
*/
OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
- OpenstackNetworkingConfig config) {
+ OpenstackNetworkingConfig config, ScalableGatewayService gatewayService) {
this.packetService = packetService;
this.openstackService = checkNotNull(openstackService);
this.config = checkNotNull(config);
+ this.gatewayService = gatewayService;
}
/**
@@ -79,10 +84,12 @@
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.build();
- packetService.requestPackets(arpSelector,
- PacketPriority.CONTROL,
- appId,
- Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
+ getExternalInfo().forEach(deviceId ->
+ packetService.requestPackets(arpSelector,
+ PacketPriority.CONTROL,
+ appId,
+ Optional.of(deviceId))
+ );
}
/**
@@ -138,4 +145,10 @@
}
return port.macAddress();
}
+
+ private List<DeviceId> getExternalInfo() {
+ List<DeviceId> externalInfoList = Lists.newArrayList();
+ gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
+ return externalInfoList;
+ }
}
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
index 406179a..6da1523 100644
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
@@ -56,6 +56,7 @@
import org.onosproject.openstacknetworking.OpenstackRoutingService;
import org.onosproject.openstacknetworking.OpenstackSubjectFactories;
import org.onosproject.openstacknetworking.OpenstackSwitchingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.onosproject.store.serializers.KryoNamespaces;
import org.onosproject.store.service.ConsistentMap;
import org.onosproject.store.service.Serializer;
@@ -113,6 +114,8 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected StorageService storageService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ScalableGatewayService gatewayService;
private ApplicationId appId;
private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
@@ -444,8 +447,7 @@
if (context.isHandled()) {
return;
- } else if (!context.inPacket().receivedFrom().deviceId().toString()
- .equals(config.gatewayBridgeId())) {
+ } else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
return;
}
@@ -472,16 +474,19 @@
}
default:
int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
- Optional<Port> port =
- getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
- if (port.isPresent()) {
+ DeviceId deviceId = pkt.receivedFrom().deviceId();
+ Port port = null;
+ port = deviceService.getPort(deviceId,
+ gatewayService.getGatewayExternalPorts(deviceId).get(0));
+ if (port != null) {
OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
Ip4Address.valueOf(iPacket.getSourceAddress()));
l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
- portNum, openstackPort, port.get(), config));
+ portNum, openstackPort, port, config));
} else {
log.warn("There`s no external interface");
}
+
break;
}
} else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
@@ -511,6 +516,10 @@
}
+ private boolean checkGatewayNode(DeviceId deviceId) {
+ return gatewayService.getGatewayDeviceIds().contains(deviceId);
+ }
+
private void clearPortNumMap() {
tpPortNumMap.entrySet().forEach(e -> {
if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
@@ -600,21 +609,12 @@
return;
}
- checkNotNull(config.physicalRouterMac());
- checkNotNull(config.gatewayBridgeId());
- checkNotNull(config.gatewayExternalInterfaceMac());
- checkNotNull(config.gatewayExternalInterfaceName());
-
- log.warn("Configured info: {}, {}, {}, {}", config.physicalRouterMac(), config.gatewayBridgeId(),
- config.gatewayExternalInterfaceMac(), config.gatewayExternalInterfaceName());
-
- rulePopulator = new OpenstackRoutingRulePopulator(appId,
- openstackService, flowObjectiveService, deviceService, driverService, config);
-
+ rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
+ deviceService, driverService, config, gatewayService);
openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService,
- openstackService, config, openstackSwitchingService);
- openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, config);
-
+ openstackService, config, openstackSwitchingService, gatewayService);
+ openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService,
+ config, gatewayService);
openstackIcmpHandler.requestPacket(appId);
openstackArpHandler.requestPacket(appId);
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
index 88bc2ce..57444d0 100644
--- a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
@@ -15,6 +15,7 @@
*/
package org.onosproject.openstacknetworking.routing;
+import com.google.common.collect.Lists;
import org.onlab.packet.Ethernet;
import org.onlab.packet.IPv4;
import org.onlab.packet.Ip4Address;
@@ -25,6 +26,7 @@
import org.onlab.packet.TpPort;
import org.onlab.packet.UDP;
import org.onosproject.core.ApplicationId;
+import org.onosproject.core.GroupId;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Port;
@@ -55,6 +57,7 @@
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,6 +80,7 @@
private final DeviceService deviceService;
private final DriverService driverService;
private final OpenstackNetworkingConfig config;
+ private final ScalableGatewayService gatewayService;
private static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
private static final String PORTNAME = "portName";
@@ -84,6 +88,7 @@
private static final String PORTNOTNULL = "Port can not be null";
private static final String DEVICENOTNULL = "Device can not be null";
+ private static final String EXTPORTNOTNULL = "External port can not be null";
private static final String TUNNEL_DESTINATION = "tunnelDst";
private static final int ROUTING_RULE_PRIORITY = 25000;
private static final int FLOATING_RULE_PRIORITY = 42000;
@@ -107,16 +112,19 @@
* @param deviceService DeviceService
* @param driverService DriverService
* @param config Configuration for openstack environment
+ * @param gatewayService scalable gateway service
*/
public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackInterfaceService openstackService,
FlowObjectiveService flowObjectiveService, DeviceService deviceService,
- DriverService driverService, OpenstackNetworkingConfig config) {
+ DriverService driverService, OpenstackNetworkingConfig config,
+ ScalableGatewayService gatewayService) {
this.appId = appId;
this.flowObjectiveService = flowObjectiveService;
this.openstackService = checkNotNull(openstackService);
this.deviceService = deviceService;
this.driverService = driverService;
this.config = config;
+ this.gatewayService = gatewayService;
}
/**
@@ -155,9 +163,6 @@
.matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
- tBuilder.setEthSrc(externalInterface)
- .setEthDst(externalRouter)
- .setIpSrc(externalIp);
switch (iPacket.getProtocol()) {
case IPv4.PROTOCOL_TCP:
@@ -177,19 +182,20 @@
break;
}
- Port port = checkNotNull(getPortOfExternalInterface(), PORTNOTNULL);
- tBuilder.setOutput(port.number());
+ tBuilder.setIpSrc(externalIp);
+ gatewayService.getGatewayNodes().forEach(node -> {
+ tBuilder.setOutput(gatewayService.getGatewayExternalPorts(node.getGatewayDeviceId()).get(0));
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(PNAT_RULE_PRIORITY)
+ .makeTemporary(PNAT_TIMEOUT)
+ .fromApp(appId)
+ .add();
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(PNAT_RULE_PRIORITY)
- .makeTemporary(PNAT_TIMEOUT)
- .fromApp(appId)
- .add();
-
- flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
+ flowObjectiveService.forward(node.getGatewayDeviceId(), fo);
+ });
}
@@ -202,7 +208,6 @@
private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp) {
IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
- DeviceId deviceId = inboundPacket.receivedFrom().deviceId();
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
@@ -232,19 +237,29 @@
break;
}
- tBuilder.extension(buildNiciraExtenstion(deviceId, getHostIpfromOpenstackPort(openstackPort)), deviceId)
- .setOutput(getTunnelPort(deviceId));
+ getGatewayNodeList().forEach(node -> {
+ DeviceId deviceId = node.id();
+ tBuilder.extension(buildNiciraExtenstion(deviceId, getHostIpfromOpenstackPort(openstackPort)), deviceId)
+ .setOutput(getTunnelPort(deviceId));
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(PNAT_RULE_PRIORITY)
- .makeTemporary(PNAT_TIMEOUT)
- .fromApp(appId)
- .add();
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(PNAT_RULE_PRIORITY)
+ .makeTemporary(PNAT_TIMEOUT)
+ .fromApp(appId)
+ .add();
- flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
+ flowObjectiveService.forward(deviceId, fo);
+ });
+ }
+
+ private List<Device> getGatewayNodeList() {
+ List<Device> devices = Lists.newArrayList();
+ gatewayService.getGatewayDeviceIds().forEach(deviceId ->
+ devices.add(checkNotNull(deviceService.getDevice(deviceId))));
+ return devices;
}
private Ip4Address getHostIpfromOpenstackPort(OpenstackPort openstackPort) {
@@ -255,15 +270,15 @@
private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) {
String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
- .filter(d -> findPortinDevice(d, openstackPortName))
+ .filter(d -> findPortinDevice(d.id(), openstackPortName))
.iterator()
.next();
checkNotNull(device, DEVICENOTNULL);
return device;
}
- private boolean findPortinDevice(Device d, String openstackPortName) {
- Port port = deviceService.getPorts(d.id())
+ private boolean findPortinDevice(DeviceId deviceId, String openstackPortName) {
+ Port port = deviceService.getPorts(deviceId)
.stream()
.filter(p -> p.isEnabled() && p.annotations().value(PORTNAME).equals(openstackPortName))
.findAny()
@@ -274,13 +289,13 @@
/**
* Builds Nicira extension for tagging remoteIp of vxlan.
*
- * @param id Device Id of vxlan source device
+ * @param deviceId Device Id of vxlan source device
* @param hostIp Remote Ip of vxlan destination device
* @return NiciraExtension Treatment
*/
- public ExtensionTreatment buildNiciraExtenstion(DeviceId id, Ip4Address hostIp) {
- Driver driver = driverService.getDriver(id);
- DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, id));
+ public ExtensionTreatment buildNiciraExtenstion(DeviceId deviceId, Ip4Address hostIp) {
+ Driver driver = driverService.getDriver(deviceId);
+ DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, deviceId));
ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
ExtensionTreatment extensionInstruction =
@@ -329,7 +344,6 @@
}
private void populateRuleGatewaytoController(long vni) {
- Device gatewayDevice = getGatewayNode();
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -346,26 +360,24 @@
.fromApp(appId)
.add();
- flowObjectiveService.forward(gatewayDevice.id(), fo);
+ getGatewayNodeList().forEach(device -> flowObjectiveService.forward(device.id(), fo));
}
private void populateComputeNodeRules(long vni) {
- Device gatewayDevice = getGatewayNode();
-
StreamSupport.stream(deviceService.getDevices().spliterator(), false)
.filter(d -> !checkGatewayNode(d.id()))
- .forEach(d -> populateRuleToGateway(d, gatewayDevice, vni));
+ .forEach(d -> populateRuleToGatewayBySgw(d.id(),
+ gatewayService.getGroupIdForGatewayLoadBalance(d.id()), vni));
}
- private void populateRuleToGateway(Device d, Device gatewayDevice, long vni) {
+ private void populateRuleToGatewayBySgw(DeviceId deviceId, GroupId groupId, long vni) {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchTunnelId(vni)
.matchEthDst(GATEWAYMAC);
- tBuilder.extension(buildNiciraExtenstion(d.id(), config.nodes().get(gatewayDevice.id())), d.id())
- .setOutput(getTunnelPort(d.id()));
+ tBuilder.group(groupId);
ForwardingObjective fo = DefaultForwardingObjective.builder()
.withSelector(sBuilder.build())
@@ -375,7 +387,28 @@
.fromApp(appId)
.add();
- flowObjectiveService.forward(d.id(), fo);
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void populateRuleToGateway(DeviceId deviceId, Device gatewayDevice, long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchEthDst(GATEWAYMAC);
+ tBuilder.extension(buildNiciraExtenstion(deviceId, config.nodes().get(gatewayDevice.id())), deviceId)
+ .setOutput(getTunnelPort(deviceId));
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
}
private Device getGatewayNode() {
@@ -383,7 +416,7 @@
}
private boolean checkGatewayNode(DeviceId deviceId) {
- return deviceId.toString().equals(config.gatewayBridgeId());
+ return gatewayService.getGatewayDeviceIds().stream().anyMatch(dId -> dId.equals(deviceId));
}
private long getVni(String netId) {
@@ -412,7 +445,7 @@
}
- private void removeRule(DeviceId id, TrafficSelector.Builder sBuilder,
+ private void removeRule(DeviceId deviceId, TrafficSelector.Builder sBuilder,
ForwardingObjective.Flag flag, int priority) {
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -424,7 +457,7 @@
.fromApp(appId)
.remove();
- flowObjectiveService.forward(id, fo);
+ flowObjectiveService.forward(deviceId, fo);
}
/**
@@ -442,8 +475,6 @@
private void populateFloatingIpIncomingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
DeviceId portDeviceId = getDevicefromOpenstackPort(port).id();
- Device gatewayNode = getGatewayNode();
- Device portNode = deviceService.getDevice(portDeviceId);
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -451,29 +482,28 @@
sBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
- tBuilder.setEthSrc(GATEWAYMAC)
- .setEthDst(port.macAddress())
- .setIpDst(floatingIP.fixedIpAddress())
- .setTunnelId(getVni(port.networkId()))
- .extension(buildNiciraExtenstion(gatewayNode.id(),
- config.nodes().get(portNode.id())), gatewayNode.id())
- .setOutput(getTunnelPort(gatewayNode.id()));
+ getGatewayNodeList().forEach(device -> {
+ DeviceId deviceId = device.id();
+ tBuilder.setEthSrc(GATEWAYMAC)
+ .setEthDst(port.macAddress())
+ .setIpDst(floatingIP.fixedIpAddress())
+ .setTunnelId(getVni(port.networkId()))
+ .extension(buildNiciraExtenstion(deviceId, config.nodes().get(portDeviceId)), deviceId)
+ .setOutput(getTunnelPort(deviceId));
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(FLOATING_RULE_PRIORITY)
- .fromApp(appId)
- .add();
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(FLOATING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
- flowObjectiveService.forward(getGatewayNode().id(), fo);
-
+ flowObjectiveService.forward(deviceId, fo);
+ });
}
private void populateFloatingIpOutgoingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
- Port outputPort = checkNotNull(getPortOfExternalInterface(), PORTNOTNULL);
-
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -481,20 +511,25 @@
.matchTunnelId(getVni(port.networkId()))
.matchIPSrc(IpPrefix.valueOf(floatingIP.fixedIpAddress(), 32));
- tBuilder.setIpSrc(floatingIP.floatingIpAddress())
- .setEthSrc(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
- .setEthDst(MacAddress.valueOf(config.physicalRouterMac()))
- .setOutput(outputPort.number());
+ getGatewayNodeList().forEach(device -> {
+ DeviceId deviceId = device.id();
+ tBuilder.setIpSrc(floatingIP.floatingIpAddress())
+ .setOutput(getExternalPortNum(deviceId));
- ForwardingObjective fo = DefaultForwardingObjective.builder()
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withFlag(ForwardingObjective.Flag.VERSATILE)
- .withPriority(FLOATING_RULE_PRIORITY)
- .fromApp(appId)
- .add();
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(FLOATING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
- flowObjectiveService.forward(getGatewayNode().id(), fo);
+ flowObjectiveService.forward(deviceId, fo);
+ });
+ }
+
+ private PortNumber getExternalPortNum(DeviceId deviceId) {
+ return checkNotNull(gatewayService.getGatewayExternalPorts(deviceId).get(0), PORTNOTNULL);
}
/**
@@ -514,38 +549,39 @@
sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
- removeRule(getGatewayNode().id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
- removeRule(getGatewayNode().id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+ getGatewayNodeList().forEach(device -> {
+ removeRule(device.id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+ removeRule(device.id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+ });
}
/**
* Populates L3 rules for east to west traffic.
*
- * @param p target VM
+ * @param openstackPort target VM
* @param targetList target openstackRouterInterfaces
*/
- public void populateL3Rules(OpenstackPort p, List<OpenstackRouterInterface> targetList) {
- Device device = getDevicefromOpenstackPort(p);
- Port port = getPortFromOpenstackPort(device, p);
- Ip4Address vmIp = (Ip4Address) p.fixedIps().values().iterator().next();
+ public void populateL3Rules(OpenstackPort openstackPort, List<OpenstackRouterInterface> targetList) {
+ Device device = getDevicefromOpenstackPort(openstackPort);
+ Port port = getPortFromOpenstackPort(device, openstackPort);
+ Ip4Address vmIp = openstackPort.fixedIps().values().iterator().next();
if (port == null) {
return;
}
targetList.forEach(routerInterface -> {
- OpenstackPort openstackPort = openstackService.port(routerInterface.portId());
- long vni = getVni(openstackPort.networkId());
+ long vni = getVni(openstackService.port(routerInterface.portId()).networkId());
if (vmIp == null) {
return;
}
- populateL3RulestoSameNode(vmIp, p, port, device, vni);
+ populateL3RulestoSameNode(vmIp, openstackPort, port, device, vni);
deviceService.getAvailableDevices().forEach(d -> {
- if (!d.equals(device) && !d.equals(getGatewayNode())) {
- populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(p));
+ if (!d.equals(device) && !checkGatewayNode(d.id())) {
+ populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(openstackPort));
}
});
@@ -618,7 +654,7 @@
OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
deviceService.getAvailableDevices().forEach(d -> {
- if (!d.equals(getGatewayNode())) {
+ if (!checkGatewayNode(d.id())) {
routerInterfaces.forEach(routerInterface -> {
String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
long vni = getVni(networkId);