CORD-537 Added public IP gateways for ARP proxy
- Added public IP gateway and MAC pairs to network config for ARP proxy
- Added vSG as a ONOS host
Change-Id: Ia722ba3843297cec7134da5d64bbf188c22762f8
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
index e071550..4672912 100644
--- a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
@@ -16,6 +16,7 @@
package org.onosproject.cordvtn;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
@@ -156,7 +157,7 @@
private HostProviderService hostProvider;
private CordVtnRuleInstaller ruleInstaller;
private CordVtnArpProxy arpProxy;
- private volatile MacAddress gatewayMac = MacAddress.NONE;
+ private volatile MacAddress privateGatewayMac = MacAddress.NONE;
/**
* Creates an cordvtn host location provider.
@@ -305,7 +306,7 @@
@Override
public void updateVirtualSubscriberGateways(HostId vSgHostId, String serviceVlan,
- Set<IpAddress> vSgIps) {
+ Map<IpAddress, MacAddress> vSgs) {
Host vSgVm = hostService.getHost(vSgHostId);
if (vSgVm == null || !vSgVm.annotations().value(S_TAG).equals(serviceVlan)) {
@@ -313,8 +314,45 @@
return;
}
- log.info("Updates vSGs in {} with {}", vSgVm.id(), vSgIps.toString());
- ruleInstaller.populateSubscriberGatewayRules(vSgVm, vSgIps);
+ log.info("Updates vSGs in {} with {}", vSgVm.id(), vSgs.toString());
+ vSgs.entrySet().stream()
+ .forEach(entry -> addVirtualSubscriberGateway(
+ vSgVm,
+ entry.getKey(),
+ entry.getValue(),
+ serviceVlan));
+
+ ruleInstaller.populateSubscriberGatewayRules(vSgVm, vSgs.keySet());
+ }
+
+ /**
+ * Adds virtual subscriber gateway to the system.
+ *
+ * @param vSgHost host virtual machine of this vSG
+ * @param vSgIp vSG ip address
+ * @param vSgMac vSG mac address
+ * @param serviceVlan service vlan
+ */
+ public void addVirtualSubscriberGateway(Host vSgHost, IpAddress vSgIp, MacAddress vSgMac, String serviceVlan) {
+ HostId hostId = HostId.hostId(vSgMac);
+ Host host = hostService.getHost(hostId);
+ if (host != null) {
+ log.debug("vSG with {} already exists", vSgMac.toString());
+ return;
+ }
+
+ log.info("vSG with IP({}) MAC({}) detected", vSgIp.toString(), vSgMac.toString());
+ DefaultAnnotations.Builder annotations = DefaultAnnotations.builder()
+ .set(S_TAG, serviceVlan);
+
+ HostDescription hostDesc = new DefaultHostDescription(
+ vSgMac,
+ VlanId.NONE,
+ vSgHost.location(),
+ Sets.newHashSet(vSgIp),
+ annotations.build());
+
+ hostProvider.hostDetected(hostId, hostDesc, false);
}
/**
@@ -438,24 +476,24 @@
* Returns public ip addresses of vSGs running inside a give vSG host.
*
* @param vSgHost vSG host
- * @return set of ip address, or empty set
+ * @return map of ip and mac address, or empty map
*/
- private Set<IpAddress> getSubscriberGatewayIps(Host vSgHost) {
+ private Map<IpAddress, MacAddress> getSubscriberGateways(Host vSgHost) {
String vPortId = vSgHost.annotations().value(OPENSTACK_PORT_ID);
String serviceVlan = vSgHost.annotations().value(S_TAG);
OpenstackPort vPort = openstackService.port(vPortId);
if (vPort == null) {
log.warn("Failed to get OpenStack port {} for VM {}", vPortId, vSgHost.id());
- return Sets.newHashSet();
+ return Maps.newHashMap();
}
if (!serviceVlan.equals(getServiceVlan(vPort))) {
log.error("Host({}) s-tag does not match with vPort s-tag", vSgHost.id());
- return Sets.newHashSet();
+ return Maps.newHashMap();
}
- return vPort.allowedAddressPairs().keySet();
+ return vPort.allowedAddressPairs();
}
/**
@@ -485,6 +523,11 @@
*/
private void serviceVmAdded(Host host) {
String vNetId = host.annotations().value(SERVICE_ID);
+ if (vNetId == null) {
+ // ignore this host, it not a VM we injected or a vSG
+ return;
+ }
+
OpenstackNetwork vNet = openstackService.network(vNetId);
if (vNet == null) {
log.warn("Failed to get OpenStack network {} for VM {}({}).",
@@ -509,19 +552,28 @@
} else {
// TODO check if the service needs an update on its group buckets after done CORD-433
ruleInstaller.updateServiceGroup(service);
- arpProxy.addServiceIp(service.serviceIp());
+ arpProxy.addGateway(service.serviceIp(), privateGatewayMac);
// sends gratuitous ARP here for the case of adding existing VMs
// when ONOS or cordvtn app is restarted
- arpProxy.sendGratuitousArp(service.serviceIp(), gatewayMac, Sets.newHashSet(host));
+ arpProxy.sendGratuitousArpForGateway(service.serviceIp(), Sets.newHashSet(host));
}
registerDhcpLease(host, service);
ruleInstaller.populateBasicConnectionRules(host, getTunnelIp(host), vNet);
- if (host.annotations().value(S_TAG) != null) {
+ String serviceVlan = host.annotations().value(S_TAG);
+ if (serviceVlan != null) {
log.debug("vSG VM detected {}", host.id());
- ruleInstaller.populateSubscriberGatewayRules(host, getSubscriberGatewayIps(host));
+ Map<IpAddress, MacAddress> vSgs = getSubscriberGateways(host);
+ vSgs.entrySet().stream()
+ .forEach(entry -> addVirtualSubscriberGateway(
+ host,
+ entry.getKey(),
+ entry.getValue(),
+ serviceVlan));
+
+ ruleInstaller.populateSubscriberGatewayRules(host, vSgs.keySet());
}
}
@@ -566,7 +618,7 @@
ruleInstaller.updateServiceGroup(service);
if (getHostsWithOpenstackNetwork(vNet).isEmpty()) {
- arpProxy.removeServiceIp(service.serviceIp());
+ arpProxy.removeGateway(service.serviceIp());
}
}
}
@@ -575,14 +627,17 @@
* Sets service network gateway MAC address and sends out gratuitous ARP to all
* VMs to update the gateway MAC address.
*
- * @param mac mac address
+ * @param newMac mac address to update
*/
- private void setServiceGatewayMac(MacAddress mac) {
- if (mac != null && !mac.equals(gatewayMac)) {
- gatewayMac = mac;
- log.debug("Set service gateway MAC address to {}", gatewayMac.toString());
+ private void setPrivateGatewayMac(MacAddress newMac) {
+ if (newMac == null || newMac.equals(privateGatewayMac)) {
+ // no updates, do nothing
+ return;
}
+ privateGatewayMac = newMac;
+ log.debug("Set service gateway MAC address to {}", privateGatewayMac.toString());
+
// TODO get existing service list from XOS and replace the loop below
Set<String> vNets = Sets.newHashSet();
hostService.getHosts().forEach(host -> vNets.add(host.annotations().value(SERVICE_ID)));
@@ -591,15 +646,29 @@
vNets.stream().forEach(vNet -> {
CordService service = getCordService(CordServiceId.of(vNet));
if (service != null) {
- arpProxy.sendGratuitousArp(
- service.serviceIp(),
- gatewayMac,
- service.hosts().keySet());
+ arpProxy.addGateway(service.serviceIp(), privateGatewayMac);
+ arpProxy.sendGratuitousArpForGateway(service.serviceIp(), service.hosts().keySet());
}
});
}
/**
+ * Sets public gateway MAC address.
+ *
+ * @param publicGateways gateway ip and mac address pairs
+ */
+ private void setPublicGatewayMac(Map<IpAddress, MacAddress> publicGateways) {
+ publicGateways.entrySet()
+ .stream()
+ .forEach(entry -> {
+ arpProxy.addGateway(entry.getKey(), entry.getValue());
+ log.debug("Added public gateway IP {}, MAC {}",
+ entry.getKey().toString(), entry.getValue().toString());
+ });
+ // TODO notice gateway MAC change to VMs holds this gateway IP
+ }
+
+ /**
* Updates configurations.
*/
private void readConfiguration() {
@@ -609,7 +678,8 @@
return;
}
- setServiceGatewayMac(config.gatewayMac());
+ setPrivateGatewayMac(config.privateGatewayMac());
+ setPublicGatewayMac(config.publicGateways());
}
private class InternalHostListener implements HostListener {
@@ -644,7 +714,7 @@
return;
}
- arpProxy.processArpPacket(context, ethPacket, gatewayMac);
+ arpProxy.processArpPacket(context, ethPacket);
}
}
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnArpProxy.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnArpProxy.java
index feee5a7..9d20e55 100644
--- a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnArpProxy.java
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnArpProxy.java
@@ -15,7 +15,7 @@
*/
package org.onosproject.cordvtn;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
import org.onlab.packet.ARP;
import org.onlab.packet.EthType;
import org.onlab.packet.Ethernet;
@@ -36,10 +36,10 @@
import org.slf4j.Logger;
import java.nio.ByteBuffer;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.slf4j.LoggerFactory.getLogger;
@@ -53,7 +53,7 @@
private final PacketService packetService;
private final HostService hostService;
- private Set<Ip4Address> serviceIPs = Sets.newHashSet();
+ private final Map<Ip4Address, MacAddress> gateways = Maps.newConcurrentMap();
/**
* Default constructor.
@@ -96,23 +96,25 @@
}
/**
- * Adds a given service IP address to be served.
+ * Adds a given gateway IP and MAC address to this ARP proxy.
*
- * @param serviceIp service ip
+ * @param gatewayIp gateway ip address
+ * @param gatewayMac gateway mac address
*/
- public void addServiceIp(IpAddress serviceIp) {
- checkNotNull(serviceIp);
- serviceIPs.add(serviceIp.getIp4Address());
+ public void addGateway(IpAddress gatewayIp, MacAddress gatewayMac) {
+ checkNotNull(gatewayIp);
+ checkNotNull(gatewayMac);
+ gateways.put(gatewayIp.getIp4Address(), gatewayMac);
}
/**
* Removes a given service IP address from this ARP proxy.
*
- * @param serviceIp service ip
+ * @param gatewayIp gateway ip address
*/
- public void removeServiceIp(IpAddress serviceIp) {
- checkNotNull(serviceIp);
- serviceIPs.remove(serviceIp.getIp4Address());
+ public void removeGateway(IpAddress gatewayIp) {
+ checkNotNull(gatewayIp);
+ gateways.remove(gatewayIp.getIp4Address());
}
/**
@@ -123,27 +125,28 @@
*
* @param context packet context
* @param ethPacket ethernet packet
- * @param gatewayMac gateway mac address
*/
- public void processArpPacket(PacketContext context, Ethernet ethPacket, MacAddress gatewayMac) {
+ public void processArpPacket(PacketContext context, Ethernet ethPacket) {
ARP arpPacket = (ARP) ethPacket.getPayload();
if (arpPacket.getOpCode() != ARP.OP_REQUEST) {
return;
}
Ip4Address targetIp = Ip4Address.valueOf(arpPacket.getTargetProtocolAddress());
- MacAddress macAddr = serviceIPs.contains(targetIp) ?
- gatewayMac : getMacFromHostService(targetIp);
- if (macAddr.equals(MacAddress.NONE)) {
+ MacAddress gatewayMac = gateways.get(targetIp);
+ MacAddress replyMac = gatewayMac != null ? gatewayMac : getMacFromHostService(targetIp);
+
+ if (replyMac.equals(MacAddress.NONE)) {
log.debug("Failed to find MAC for {}", targetIp.toString());
context.block();
return;
}
+ log.trace("Send ARP reply for {} with {}", targetIp.toString(), replyMac.toString());
Ethernet ethReply = ARP.buildArpReply(
targetIp,
- macAddr,
+ replyMac,
ethPacket);
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
@@ -161,14 +164,17 @@
/**
* Emits gratuitous ARP when a gateway mac address has been changed.
*
- * @param ip ip address to update MAC
- * @param mac new mac address
+ * @param gatewayIp gateway ip address to update MAC
* @param hosts set of hosts to send gratuitous ARP packet
*/
- public void sendGratuitousArp(IpAddress ip, MacAddress mac, Set<Host> hosts) {
- checkArgument(!mac.equals(MacAddress.NONE));
+ public void sendGratuitousArpForGateway(IpAddress gatewayIp, Set<Host> hosts) {
+ MacAddress gatewayMac = gateways.get(gatewayIp.getIp4Address());
+ if (gatewayMac == null) {
+ log.debug("Gateway {} is not registered to ARP proxy", gatewayIp.toString());
+ return;
+ }
- Ethernet ethArp = buildGratuitousArp(ip.getIp4Address(), mac);
+ Ethernet ethArp = buildGratuitousArp(gatewayIp.getIp4Address(), gatewayMac);
hosts.stream().forEach(host -> {
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
.setOutput(host.location().port())
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnConfig.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnConfig.java
index 963019f..ec83432 100644
--- a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnConfig.java
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnConfig.java
@@ -16,7 +16,9 @@
package org.onosproject.cordvtn;
import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
import org.onlab.packet.TpPort;
import org.onosproject.core.ApplicationId;
@@ -24,6 +26,7 @@
import org.onosproject.net.config.Config;
import org.slf4j.Logger;
+import java.util.Map;
import java.util.Set;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -36,6 +39,9 @@
protected final Logger log = getLogger(getClass());
+ public static final String PRIVATE_GATEWAY_MAC = "privateGatewayMac";
+ public static final String PUBLIC_GATEWAYS = "publicGateways";
+ public static final String GATEWAY_IP = "gatewayIp";
public static final String GATEWAY_MAC = "gatewayMac";
public static final String LOCAL_MANAGEMENT_IP = "localManagementIp";
public static final String OVSDB_PORT = "ovsdbPort";
@@ -80,12 +86,12 @@
}
/**
- * Returns gateway MAC address.
+ * Returns private network gateway MAC address.
*
* @return mac address, or null
*/
- public MacAddress gatewayMac() {
- JsonNode jsonNode = object.get(GATEWAY_MAC);
+ public MacAddress privateGatewayMac() {
+ JsonNode jsonNode = object.get(PRIVATE_GATEWAY_MAC);
if (jsonNode == null) {
return null;
}
@@ -99,6 +105,31 @@
}
/**
+ * Returns public network gateway IP and MAC address pairs.
+ *
+ * @return map of ip and mac address
+ */
+ public Map<IpAddress, MacAddress> publicGateways() {
+ JsonNode jsonNodes = object.get(PUBLIC_GATEWAYS);
+ if (jsonNodes == null) {
+ return null;
+ }
+
+ Map<IpAddress, MacAddress> publicGateways = Maps.newHashMap();
+ jsonNodes.forEach(jsonNode -> {
+ try {
+ publicGateways.put(
+ IpAddress.valueOf(jsonNode.path(GATEWAY_IP).asText()),
+ MacAddress.valueOf(jsonNode.path(GATEWAY_MAC).asText()));
+ } catch (IllegalArgumentException | NullPointerException e) {
+ log.error("Wrong address format {}", e.toString());
+ }
+ });
+
+ return publicGateways;
+ }
+
+ /**
* Returns local management network address.
*
* @return network address
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnService.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnService.java
index ead644f..f133c44 100644
--- a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnService.java
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnService.java
@@ -16,10 +16,11 @@
package org.onosproject.cordvtn;
import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
import org.onosproject.net.ConnectPoint;
import org.onosproject.net.HostId;
-import java.util.Set;
+import java.util.Map;
/**
* Service for provisioning overlay virtual networks on compute nodes.
@@ -67,8 +68,8 @@
*
* @param vSgHost host id of vSG host
* @param serviceVlan service vlan id
- * @param vSgIps set of ip address of vSGs running in this vSG host
+ * @param vSgs map of ip and mac address of vSGs running in this vSG host
*/
void updateVirtualSubscriberGateways(HostId vSgHost, String serviceVlan,
- Set<IpAddress> vSgIps);
+ Map<IpAddress, MacAddress> vSgs);
}
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/rest/NeutronMl2PortsWebResource.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/rest/NeutronMl2PortsWebResource.java
index cd8f555..21bbda4 100644
--- a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/rest/NeutronMl2PortsWebResource.java
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/rest/NeutronMl2PortsWebResource.java
@@ -17,7 +17,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
import org.onosproject.cordvtn.CordVtnService;
@@ -36,7 +36,7 @@
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.InputStream;
-import java.util.Set;
+import java.util.Map;
/**
@@ -88,16 +88,17 @@
// this is allowed address pairs updates
MacAddress mac = MacAddress.valueOf(jsonNode.path(MAC_ADDRESS).asText());
- Set<IpAddress> vSgIps = Sets.newHashSet();
+ Map<IpAddress, MacAddress> vSgs = Maps.newHashMap();
jsonNode.path(ADDRESS_PAIRS).forEach(addrPair -> {
- IpAddress ip = IpAddress.valueOf(addrPair.path(IP_ADDERSS).asText());
- vSgIps.add(ip);
+ IpAddress pairIp = IpAddress.valueOf(addrPair.path(IP_ADDERSS).asText());
+ MacAddress pairMac = MacAddress.valueOf(addrPair.path(MAC_ADDRESS).asText());
+ vSgs.put(pairIp, pairMac);
});
service.updateVirtualSubscriberGateways(
HostId.hostId(mac),
name.substring(STAG_BEGIN_INDEX),
- vSgIps);
+ vSgs);
} catch (Exception e) {
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
}