Refactor OpenStack SNAT from RoutingHandler to RoutingSnatHandler
Change-Id: Id2d8fabebc5b1d8b9f52f089dc2dbc9bc8dbce97
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingHandler.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingHandler.java
index d1864d7..96a2c5b 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingHandler.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingHandler.java
@@ -17,35 +17,24 @@
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
import org.onlab.packet.Ethernet;
import org.onlab.packet.IPv4;
import org.onlab.packet.IpAddress;
import org.onlab.packet.IpPrefix;
-import org.onlab.packet.MacAddress;
import org.onlab.packet.VlanId;
-import org.onlab.util.Tools;
-import org.onosproject.cfg.ComponentConfigService;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
-import org.onosproject.mastership.MastershipService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.driver.DriverService;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
-import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.openstacknetworking.api.Constants;
-import org.onosproject.openstacknetworking.api.ExternalPeerRouter;
-import org.onosproject.openstacknetworking.api.InstancePort;
-import org.onosproject.openstacknetworking.api.InstancePortEvent;
-import org.onosproject.openstacknetworking.api.InstancePortListener;
import org.onosproject.openstacknetworking.api.InstancePortService;
import org.onosproject.openstacknetworking.api.OpenstackFlowRuleService;
import org.onosproject.openstacknetworking.api.OpenstackNetwork.Type;
@@ -53,55 +42,40 @@
import org.onosproject.openstacknetworking.api.OpenstackRouterEvent;
import org.onosproject.openstacknetworking.api.OpenstackRouterListener;
import org.onosproject.openstacknetworking.api.OpenstackRouterService;
-import org.onosproject.openstacknetworking.util.RulePopulatorUtil;
import org.onosproject.openstacknode.api.OpenstackNode;
import org.onosproject.openstacknode.api.OpenstackNodeEvent;
import org.onosproject.openstacknode.api.OpenstackNodeListener;
import org.onosproject.openstacknode.api.OpenstackNodeService;
-import org.openstack4j.model.network.ExternalGateway;
import org.openstack4j.model.network.Network;
import org.openstack4j.model.network.Router;
import org.openstack4j.model.network.RouterInterface;
import org.openstack4j.model.network.Subnet;
-import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
-import org.osgi.service.component.annotations.Modified;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Dictionary;
import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static org.onlab.util.Tools.groupedThreads;
-import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_EXTERNAL_ROUTER_MAC;
-import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_GATEWAY_MAC;
-import static org.onosproject.openstacknetworking.api.Constants.GW_COMMON_TABLE;
import static org.onosproject.openstacknetworking.api.Constants.OPENSTACK_NETWORKING_APP_ID;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ADMIN_RULE;
-import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ICMP_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_INTERNAL_ROUTING_RULE;
-import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_SWITCHING_RULE;
import static org.onosproject.openstacknetworking.api.Constants.ROUTING_TABLE;
import static org.onosproject.openstacknetworking.api.Constants.STAT_OUTBOUND_TABLE;
-import static org.onosproject.openstacknetworking.api.InstancePort.State.ACTIVE;
-import static org.onosproject.openstacknetworking.api.OpenstackNetwork.Type.FLAT;
import static org.onosproject.openstacknetworking.api.OpenstackNetwork.Type.GENEVE;
import static org.onosproject.openstacknetworking.api.OpenstackNetwork.Type.GRE;
import static org.onosproject.openstacknetworking.api.OpenstackNetwork.Type.VLAN;
import static org.onosproject.openstacknetworking.api.OpenstackNetwork.Type.VXLAN;
-import static org.onosproject.openstacknetworking.impl.OsgiPropertyConstants.USE_STATEFUL_SNAT;
-import static org.onosproject.openstacknetworking.impl.OsgiPropertyConstants.USE_STATEFUL_SNAT_DEFAULT;
import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.tunnelPortNumByNetType;
import static org.onosproject.openstacknetworking.util.RulePopulatorUtil.buildExtension;
import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.COMPUTE;
@@ -110,12 +84,7 @@
/**
* Handles OpenStack router events.
*/
-@Component(
- immediate = true,
- property = {
- USE_STATEFUL_SNAT + ":Boolean=" + USE_STATEFUL_SNAT_DEFAULT
- }
-)
+@Component(immediate = true)
public class OpenstackRoutingHandler {
private final Logger log = LoggerFactory.getLogger(getClass());
@@ -124,11 +93,6 @@
private static final String MSG_DISABLED = "Disabled ";
private static final String ERR_UNSUPPORTED_NET_TYPE = "Unsupported network type";
- private static final int VM_PREFIX = 32;
-
- /** Use Stateful SNAT for source NATing. */
- private boolean useStatefulSnat = USE_STATEFUL_SNAT_DEFAULT;
-
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -156,20 +120,10 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected OpenstackFlowRuleService osFlowRuleService;
- @Reference(cardinality = ReferenceCardinality.MANDATORY)
- protected MastershipService mastershipService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY)
- protected DriverService driverService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY)
- protected ComponentConfigService configService;
-
private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(
groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
private final OpenstackNodeListener osNodeListener = new InternalNodeEventListener();
private final OpenstackRouterListener osRouterListener = new InternalRouterEventListener();
- private final InstancePortListener instancePortListener = new InternalInstancePortListener();
private ApplicationId appId;
private NodeId localNodeId;
@@ -181,8 +135,6 @@
leadershipService.runForLeadership(appId.name());
osNodeService.addListener(osNodeListener);
osRouterService.addListener(osRouterListener);
- instancePortService.addListener(instancePortListener);
- configService.registerProperties(getClass());
log.info("Started");
}
@@ -191,34 +143,13 @@
protected void deactivate() {
osRouterService.removeListener(osRouterListener);
osNodeService.removeListener(osNodeListener);
- instancePortService.removeListener(instancePortListener);
leadershipService.withdraw(appId.name());
- configService.unregisterProperties(getClass(), false);
eventExecutor.shutdown();
log.info("Stopped");
}
- @Modified
- protected void modified(ComponentContext context) {
- Dictionary<?, ?> properties = context.getProperties();
- Boolean flag;
-
- flag = Tools.isPropertyEnabled(properties, USE_STATEFUL_SNAT);
- if (flag == null) {
- log.info("useStatefulSnat is not configured, " +
- "using current value of {}", useStatefulSnat);
- } else {
- useStatefulSnat = flag;
- log.info("Configured. useStatefulSnat is {}",
- useStatefulSnat ? "enabled" : "disabled");
- }
-
- resetSnatRules();
- }
-
private void routerUpdated(Router osRouter) {
- ExternalGateway exGateway = osRouter.getExternalGatewayInfo();
osRouterService.routerInterfaces(osRouter.getId()).forEach(iface -> {
Network network = osNetworkAdminService.network(
osNetworkAdminService.subnet(iface.getSubnetId())
@@ -229,46 +160,6 @@
setRouterAdminRules(network.getProviderSegID(),
netType, !osRouter.isAdminStateUp());
});
-
- ExternalPeerRouter externalPeerRouter =
- osNetworkAdminService.externalPeerRouter(exGateway);
- VlanId vlanId = externalPeerRouter == null ? VlanId.NONE : externalPeerRouter.vlanId();
-
- if (exGateway == null) {
- deleteUnassociatedExternalPeerRouter();
- osRouterService.routerInterfaces(osRouter.getId()).forEach(iface ->
- setSourceNat(iface, false));
- } else {
- osNetworkAdminService.deriveExternalPeerRouterMac(exGateway, osRouter, vlanId);
- osRouterService.routerInterfaces(osRouter.getId()).forEach(iface ->
- setSourceNat(iface, exGateway.isEnableSnat()));
- }
- }
-
- private void deleteUnassociatedExternalPeerRouter() {
- log.trace("Deleting unassociated external peer router");
-
- try {
- Set<String> routerIps = Sets.newConcurrentHashSet();
-
- osRouterService.routers().stream()
- .filter(router -> getGatewayIpAddress(router) != null)
- .map(router -> getGatewayIpAddress(router).toString())
- .forEach(routerIps::add);
-
- osNetworkAdminService.externalPeerRouters().stream()
- .filter(externalPeerRouter ->
- !routerIps.contains(externalPeerRouter.ipAddress().toString()))
- .forEach(externalPeerRouter -> {
- osNetworkAdminService
- .deleteExternalPeerRouter(
- externalPeerRouter.ipAddress().toString());
- log.trace("Deleted unassociated external peer router {}",
- externalPeerRouter.ipAddress().toString());
- });
- } catch (Exception e) {
- log.error("Exception occurred because of {}", e.toString());
- }
}
private void routerRemove(Router osRouter) {
@@ -301,10 +192,6 @@
setInternalRoutes(osRouter, osSubnet, true);
setGatewayIcmp(osSubnet, osRouter, true);
- ExternalGateway exGateway = osRouter.getExternalGatewayInfo();
- if (exGateway != null && exGateway.isEnableSnat()) {
- setSourceNat(osRouterIface, true);
- }
log.info("Connected subnet({}) to {}", osSubnet.getCidr(), osRouter.getName());
}
@@ -326,128 +213,9 @@
setInternalRoutes(osRouter, osSubnet, false);
setGatewayIcmp(osSubnet, osRouter, false);
- ExternalGateway exGateway = osRouter.getExternalGatewayInfo();
- if (exGateway != null && exGateway.isEnableSnat()) {
- setSourceNat(osRouterIface, false);
- }
log.info("Disconnected subnet({}) from {}", osSubnet.getCidr(), osRouter.getName());
}
- private void setSourceNat(RouterInterface routerIface, boolean install) {
- Subnet osSubnet = osNetworkAdminService.subnet(routerIface.getSubnetId());
- Network osNet = osNetworkAdminService.network(osSubnet.getNetworkId());
- Type netType = osNetworkAdminService.networkType(osSubnet.getNetworkId());
-
- osNodeService.completeNodes(COMPUTE).forEach(cNode -> {
- setRulesToGateway(cNode, osNet.getProviderSegID(),
- IpPrefix.valueOf(osSubnet.getCidr()), netType, install);
- });
-
- if (useStatefulSnat) {
- setStatefulSnatRules(routerIface, install);
- } else {
- setReactiveSnatRules(routerIface, install);
- }
-
- final String updateStr = install ? MSG_ENABLED : MSG_DISABLED;
- log.info(updateStr + "external access for subnet({})", osSubnet.getCidr());
- }
-
- private void setStatefulSnatRules(RouterInterface routerIface, boolean install) {
- Subnet osSubnet = osNetworkAdminService.subnet(routerIface.getSubnetId());
- Network osNet = osNetworkAdminService.network(osSubnet.getNetworkId());
- Type netType = osNetworkAdminService.networkType(osSubnet.getNetworkId());
-
- if (netType == FLAT) {
- return;
- }
-
- Optional<Router> osRouter = osRouterService.routers().stream()
- .filter(router -> osRouterService.routerInterfaces(routerIface.getId()) != null)
- .findAny();
-
- if (!osRouter.isPresent()) {
- log.error("Cannot find a router for router interface {} ", routerIface);
- return;
- }
- IpAddress natAddress = getGatewayIpAddress(osRouter.get());
- if (natAddress == null) {
- return;
- }
- String netId = osNetworkAdminService.subnet(routerIface.getSubnetId()).getNetworkId();
-
- osNodeService.completeNodes(OpenstackNode.NodeType.GATEWAY)
- .forEach(gwNode -> {
- instancePortService.instancePorts(netId)
- .stream()
- .filter(port -> port.state() == ACTIVE)
- .forEach(port -> setRulesForSnatIngressRule(gwNode.intgBridge(),
- Long.parseLong(osNet.getProviderSegID()),
- IpPrefix.valueOf(port.ipAddress(), VM_PREFIX),
- port.deviceId(),
- netType,
- install));
-
- setOvsNatIngressRule(gwNode.intgBridge(),
- IpPrefix.valueOf(natAddress, VM_PREFIX),
- Constants.DEFAULT_EXTERNAL_ROUTER_MAC, install);
- setOvsNatEgressRule(gwNode.intgBridge(),
- natAddress, Long.parseLong(osNet.getProviderSegID()),
- gwNode.patchPortNum(), install);
- });
- }
-
- private void setReactiveSnatRules(RouterInterface routerIface, boolean install) {
- Subnet osSubnet = osNetworkAdminService.subnet(routerIface.getSubnetId());
- Network osNet = osNetworkAdminService.network(osSubnet.getNetworkId());
- Type netType = osNetworkAdminService.networkType(osSubnet.getNetworkId());
-
- osNodeService.completeNodes(GATEWAY)
- .forEach(gwNode -> setRulesToController(
- gwNode.intgBridge(),
- osNet.getProviderSegID(),
- IpPrefix.valueOf(osSubnet.getCidr()),
- netType,
- install));
- }
-
- private IpAddress getGatewayIpAddress(Router osRouter) {
-
- if (osRouter.getExternalGatewayInfo() == null) {
- return null;
- }
- String extNetId = osNetworkAdminService.network(
- osRouter.getExternalGatewayInfo().getNetworkId()).getId();
- Optional<Subnet> extSubnet = osNetworkAdminService.subnets().stream()
- .filter(subnet -> subnet.getNetworkId().equals(extNetId))
- .findAny();
-
- if (!extSubnet.isPresent()) {
- log.error("Cannot find externel subnet for the router");
- return null;
- }
-
- return IpAddress.valueOf(extSubnet.get().getGateway());
- }
-
- private void resetSnatRules() {
- if (useStatefulSnat) {
- osRouterService.routerInterfaces().forEach(
- routerIface -> {
- setReactiveSnatRules(routerIface, false);
- setStatefulSnatRules(routerIface, true);
- }
- );
- } else {
- osRouterService.routerInterfaces().forEach(
- routerIface -> {
- setStatefulSnatRules(routerIface, false);
- setReactiveSnatRules(routerIface, true);
- }
- );
- }
- }
-
private void setGatewayIcmp(Subnet osSubnet, Router osRouter, boolean install) {
OpenstackNode srcNatGw = osNodeService.completeNodes(GATEWAY)
.stream().findFirst().orElse(null);
@@ -771,107 +539,6 @@
install);
}
- private void setRulesToGateway(OpenstackNode osNode,
- String segmentId,
- IpPrefix srcSubnet,
- Type networkType,
- boolean install) {
- OpenstackNode sourceNatGateway =
- osNodeService.completeNodes(GATEWAY).stream().findFirst().orElse(null);
-
- if (sourceNatGateway == null) {
- return;
- }
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPSrc(srcSubnet.getIp4Prefix())
- .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
-
- switch (networkType) {
- case VXLAN:
- case GRE:
- case GENEVE:
- sBuilder.matchTunnelId(Long.parseLong(segmentId));
- break;
- case VLAN:
- sBuilder.matchVlanId(VlanId.vlanId(segmentId));
- break;
- default:
- final String error = String.format("%s %s",
- ERR_UNSUPPORTED_NET_TYPE,
- networkType.toString());
- throw new IllegalStateException(error);
- }
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- switch (networkType) {
- case VXLAN:
- case GRE:
- case GENEVE:
- PortNumber portNum = tunnelPortNumByNetType(networkType, osNode);
- tBuilder.extension(buildExtension(
- deviceService,
- osNode.intgBridge(),
- sourceNatGateway.dataIp().getIp4Address()),
- osNode.intgBridge())
- .setOutput(portNum);
- break;
-
- case VLAN:
- tBuilder.setOutput(osNode.vlanPortNum());
- break;
-
- default:
- break;
- }
-
- osFlowRuleService.setRule(
- appId,
- osNode.intgBridge(),
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_EXTERNAL_ROUTING_RULE,
- ROUTING_TABLE,
- install);
- }
-
- private void setRulesForSnatIngressRule(DeviceId deviceId,
- Long vni,
- IpPrefix destVmIp,
- DeviceId dstDeviceId,
- Type networkType,
- boolean install) {
-
- TrafficSelector selector = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(destVmIp)
- .build();
-
- PortNumber portNum = tunnelPortNumByNetType(networkType,
- osNodeService.node(deviceId));
-
- TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- .setTunnelId(vni)
- .extension(buildExtension(
- deviceService,
- deviceId,
- osNodeService.node(dstDeviceId).dataIp().getIp4Address()),
- deviceId)
- .setOutput(portNum)
- .build();
-
- osFlowRuleService.setRule(
- appId,
- deviceId,
- selector,
- treatment,
- PRIORITY_EXTERNAL_ROUTING_RULE,
- Constants.GW_COMMON_TABLE,
- install);
- }
-
private void setRulesToGatewayWithRoutableSubnets(OpenstackNode osNode,
OpenstackNode sourceNatGateway,
String segmentId,
@@ -941,118 +608,6 @@
install);
}
- private void setOvsNatIngressRule(DeviceId deviceId,
- IpPrefix cidr,
- MacAddress dstMac,
- boolean install) {
-
- TrafficSelector selector = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(cidr)
- .build();
-
- ExtensionTreatment natTreatment = RulePopulatorUtil
- .niciraConnTrackTreatmentBuilder(driverService, deviceId)
- .commit(false)
- .natAction(true)
- .table((short) 0)
- .build();
-
- TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- .setEthDst(dstMac)
- .extension(natTreatment, deviceId)
- .build();
-
- osFlowRuleService.setRule(
- appId,
- deviceId,
- selector,
- treatment,
- PRIORITY_STATEFUL_SNAT_RULE,
- GW_COMMON_TABLE,
- install);
- }
-
- private void setOvsNatEgressRule(DeviceId deviceId,
- IpAddress natAddress,
- long vni,
- PortNumber output,
- boolean install) {
-
- TrafficSelector selector = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchEthDst(DEFAULT_GATEWAY_MAC)
- .matchTunnelId(vni)
- .build();
-
- ExtensionTreatment natTreatment = RulePopulatorUtil
- .niciraConnTrackTreatmentBuilder(driverService, deviceId)
- .commit(true)
- .natAction(true)
- .natIp(natAddress)
- .build();
-
- TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- .extension(natTreatment, deviceId)
- .setEthDst(DEFAULT_EXTERNAL_ROUTER_MAC)
- .setEthSrc(DEFAULT_GATEWAY_MAC)
- .setOutput(output)
- .build();
-
- osFlowRuleService.setRule(
- appId,
- deviceId,
- selector,
- treatment,
- PRIORITY_STATEFUL_SNAT_RULE,
- GW_COMMON_TABLE,
- install);
- }
-
- private void setRulesToController(DeviceId deviceId,
- String segmentId,
- IpPrefix srcSubnet,
- Type networkType,
- boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPSrc(srcSubnet)
- .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
-
- switch (networkType) {
- case VXLAN:
- case GRE:
- case GENEVE:
- sBuilder.matchTunnelId(Long.parseLong(segmentId));
- break;
- case VLAN:
- sBuilder.matchVlanId(VlanId.vlanId(segmentId));
- break;
- default:
- final String error = String.format("%s %s",
- ERR_UNSUPPORTED_NET_TYPE,
- networkType.toString());
- throw new IllegalStateException(error);
- }
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- if (networkType == VLAN) {
- tBuilder.popVlan();
- }
-
- tBuilder.punt();
-
- osFlowRuleService.setRule(
- appId,
- deviceId,
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_EXTERNAL_ROUTING_RULE,
- GW_COMMON_TABLE,
- install);
- }
-
private void setRouterAdminRules(String segmentId,
Type networkType,
boolean install) {
@@ -1100,7 +655,6 @@
return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
}
- // FIXME only one leader in the cluster should process
@Override
public void event(OpenstackRouterEvent event) {
switch (event.type()) {
@@ -1245,120 +799,4 @@
log.info("Reconfigure routers for {}", osNode.hostname());
}
}
-
- private class InternalInstancePortListener implements InstancePortListener {
-
- private boolean isRelevantHelper(InstancePortEvent event) {
- return mastershipService.isLocalMaster(event.subject().deviceId());
- }
-
- @Override
- public void event(InstancePortEvent event) {
- InstancePort instPort = event.subject();
- switch (event.type()) {
- case OPENSTACK_INSTANCE_PORT_DETECTED:
- case OPENSTACK_INSTANCE_PORT_UPDATED:
- eventExecutor.execute(() ->
- processInstancePortDetection(event, instPort));
- break;
- case OPENSTACK_INSTANCE_PORT_VANISHED:
- eventExecutor.execute(() ->
- processInstancePortRemoval(event, instPort));
- break;
- case OPENSTACK_INSTANCE_MIGRATION_STARTED:
- eventExecutor.execute(() ->
- processInstanceMigrationStart(event, instPort));
- break;
- case OPENSTACK_INSTANCE_MIGRATION_ENDED:
- eventExecutor.execute(() ->
- processInstanceMigrationEnd(event, instPort));
- break;
- default:
- break;
- }
- }
-
- private void processInstancePortDetection(InstancePortEvent event,
- InstancePort instPort) {
- if (!isRelevantHelper(event)) {
- return;
- }
-
- log.info("RoutingHandler: Instance port detected MAC:{} IP:{}",
- instPort.macAddress(),
- instPort.ipAddress());
-
- instPortDetected(event.subject());
- }
-
- private void processInstancePortRemoval(InstancePortEvent event,
- InstancePort instPort) {
- if (!isRelevantHelper(event)) {
- return;
- }
-
- log.info("RoutingHandler: Instance port vanished MAC:{} IP:{}",
- instPort.macAddress(),
- instPort.ipAddress());
-
- instPortRemoved(event.subject());
- }
-
- private void processInstanceMigrationStart(InstancePortEvent event,
- InstancePort instPort) {
- if (!isRelevantHelper(event)) {
- return;
- }
-
- log.info("RoutingHandler: Migration started for MAC:{} IP:{}",
- instPort.macAddress(),
- instPort.ipAddress());
-
- instPortDetected(instPort);
- }
-
- private void processInstanceMigrationEnd(InstancePortEvent event,
- InstancePort instPort) {
- log.info("RoutingHandler: Migration finished for MAC:{} IP:{}",
- instPort.macAddress(),
- instPort.ipAddress());
- // TODO: need to reconfigure rules to point to update VM
- }
-
- private void instPortDetected(InstancePort instPort) {
- Network network = osNetworkAdminService.network(instPort.networkId());
- Type netType = osNetworkAdminService.networkType(instPort.networkId());
-
- if (netType == FLAT) {
- return;
- }
-
- if (useStatefulSnat) {
- osNodeService.completeNodes(GATEWAY)
- .forEach(gwNode -> setRulesForSnatIngressRule(
- gwNode.intgBridge(),
- Long.parseLong(network.getProviderSegID()),
- IpPrefix.valueOf(instPort.ipAddress(), VM_PREFIX),
- instPort.deviceId(), netType, true));
- }
- }
-
- private void instPortRemoved(InstancePort instPort) {
- Network network = osNetworkAdminService.network(instPort.networkId());
- Type netType = osNetworkAdminService.networkType(instPort.networkId());
-
- if (netType == FLAT) {
- return;
- }
-
- if (useStatefulSnat) {
- osNodeService.completeNodes(GATEWAY)
- .forEach(gwNode -> setRulesForSnatIngressRule(
- gwNode.intgBridge(),
- Long.parseLong(network.getProviderSegID()),
- IpPrefix.valueOf(instPort.ipAddress(), VM_PREFIX),
- instPort.deviceId(), netType, false));
- }
- }
- }
}
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingSnatHandler.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingSnatHandler.java
index 9ee5f3d..2fbc09a 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingSnatHandler.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackRoutingSnatHandler.java
@@ -15,88 +15,137 @@
*/
package org.onosproject.openstacknetworking.impl;
+import com.google.common.collect.Sets;
import org.onlab.packet.Ethernet;
import org.onlab.packet.IPv4;
import org.onlab.packet.IpAddress;
import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
import org.onlab.packet.TCP;
import org.onlab.packet.TpPort;
import org.onlab.packet.UDP;
import org.onlab.packet.VlanId;
import org.onlab.util.KryoNamespace;
+import org.onlab.util.Tools;
+import org.onosproject.cfg.ComponentConfigService;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
+import org.onosproject.mastership.MastershipService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.net.packet.DefaultOutboundPacket;
import org.onosproject.net.packet.InboundPacket;
import org.onosproject.net.packet.PacketContext;
import org.onosproject.net.packet.PacketProcessor;
import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstacknetworking.api.Constants;
import org.onosproject.openstacknetworking.api.ExternalPeerRouter;
import org.onosproject.openstacknetworking.api.InstancePort;
+import org.onosproject.openstacknetworking.api.InstancePortEvent;
+import org.onosproject.openstacknetworking.api.InstancePortListener;
import org.onosproject.openstacknetworking.api.InstancePortService;
import org.onosproject.openstacknetworking.api.OpenstackFlowRuleService;
import org.onosproject.openstacknetworking.api.OpenstackNetwork.Type;
+import org.onosproject.openstacknetworking.api.OpenstackNetworkAdminService;
import org.onosproject.openstacknetworking.api.OpenstackNetworkService;
+import org.onosproject.openstacknetworking.api.OpenstackRouterEvent;
+import org.onosproject.openstacknetworking.api.OpenstackRouterListener;
import org.onosproject.openstacknetworking.api.OpenstackRouterService;
import org.onosproject.openstacknetworking.util.RulePopulatorUtil;
import org.onosproject.openstacknode.api.OpenstackNode;
+import org.onosproject.openstacknode.api.OpenstackNodeEvent;
+import org.onosproject.openstacknode.api.OpenstackNodeListener;
import org.onosproject.openstacknode.api.OpenstackNodeService;
import org.onosproject.store.serializers.KryoNamespaces;
import org.onosproject.store.service.ConsistentMap;
import org.onosproject.store.service.DistributedSet;
import org.onosproject.store.service.Serializer;
import org.onosproject.store.service.StorageService;
+import org.openstack4j.model.network.ExternalGateway;
import org.openstack4j.model.network.IP;
import org.openstack4j.model.network.Network;
import org.openstack4j.model.network.Port;
+import org.openstack4j.model.network.Router;
+import org.openstack4j.model.network.RouterInterface;
import org.openstack4j.model.network.Subnet;
+import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Modified;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.slf4j.Logger;
import java.nio.ByteBuffer;
+import java.util.Dictionary;
+import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_EXTERNAL_ROUTER_MAC;
import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_GATEWAY_MAC;
import static org.onosproject.openstacknetworking.api.Constants.GW_COMMON_TABLE;
import static org.onosproject.openstacknetworking.api.Constants.OPENSTACK_NETWORKING_APP_ID;
+import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_SNAT_RULE;
+import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
+import static org.onosproject.openstacknetworking.api.Constants.ROUTING_TABLE;
+import static org.onosproject.openstacknetworking.api.InstancePort.State.ACTIVE;
+import static org.onosproject.openstacknetworking.api.OpenstackNetwork.Type.FLAT;
+import static org.onosproject.openstacknetworking.api.OpenstackNetwork.Type.VLAN;
+import static org.onosproject.openstacknetworking.impl.OsgiPropertyConstants.USE_STATEFUL_SNAT;
+import static org.onosproject.openstacknetworking.impl.OsgiPropertyConstants.USE_STATEFUL_SNAT_DEFAULT;
import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.externalIpFromSubnet;
import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.externalPeerRouterFromSubnet;
import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.tunnelPortNumByNetType;
+import static org.onosproject.openstacknetworking.util.RulePopulatorUtil.buildExtension;
+import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.COMPUTE;
import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.GATEWAY;
import static org.slf4j.LoggerFactory.getLogger;
/**
* Handle packets needs SNAT.
*/
-@Component(immediate = true)
+@Component(
+ immediate = true,
+ property = {
+ USE_STATEFUL_SNAT + ":Boolean=" + USE_STATEFUL_SNAT_DEFAULT
+ }
+)
public class OpenstackRoutingSnatHandler {
private final Logger log = getLogger(getClass());
- private static final String ERR_PACKETIN = "Failed to handle packet in: ";
+ private static final String ERR_PACKET_IN = "Failed to handle packet in: ";
private static final String ERR_UNSUPPORTED_NET_TYPE = "Unsupported network type";
private static final long TIME_OUT_SNAT_PORT_MS = 120L * 1000L;
- private static final int TP_PORT_MINIMUM_NUM = 65000;
+ private static final int TP_PORT_MINIMUM_NUM = 30000;
private static final int TP_PORT_MAXIMUM_NUM = 65535;
private static final int VM_PREFIX = 32;
- private static final KryoNamespace.Builder NUMBER_SERIALIZER = KryoNamespace.newBuilder()
+ private static final String MSG_ENABLED = "Enabled ";
+ private static final String MSG_DISABLED = "Disabled ";
+
+ /** Use Stateful SNAT for source NATing. */
+ private boolean useStatefulSnat = USE_STATEFUL_SNAT_DEFAULT;
+
+ private static final KryoNamespace.Builder NUMBER_SERIALIZER =
+ KryoNamespace.newBuilder()
.register(KryoNamespaces.API);
@Reference(cardinality = ReferenceCardinality.MANDATORY)
@@ -115,9 +164,27 @@
protected InstancePortService instancePortService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ComponentConfigService configService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DriverService driverService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
protected OpenstackNodeService osNodeService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected OpenstackNetworkAdminService osNetworkAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
protected OpenstackNetworkService osNetworkService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
@@ -129,10 +196,14 @@
private final ExecutorService eventExecutor = newSingleThreadExecutor(
groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
private final PacketProcessor packetProcessor = new InternalPacketProcessor();
+ private final InstancePortListener instancePortListener = new InternalInstancePortListener();
+ private final OpenstackRouterListener osRouterListener = new InternalRouterEventListener();
+ private final OpenstackNodeListener osNodeListener = new InternalNodeEventListener();
private ConsistentMap<Integer, Long> allocatedPortNumMap;
private DistributedSet<Integer> unUsedPortNumSet;
private ApplicationId appId;
+ private NodeId localNodeId;
@Activate
protected void activate() {
@@ -152,27 +223,48 @@
initializeUnusedPortNumSet();
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
+
+ configService.registerProperties(getClass());
+ instancePortService.addListener(instancePortListener);
+ osRouterService.addListener(osRouterListener);
+ osNodeService.addListener(osNodeListener);
+
log.info("Started");
}
- private void initializeUnusedPortNumSet() {
- for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
- if (!allocatedPortNumMap.containsKey(i)) {
- unUsedPortNumSet.add(i);
- }
- }
-
- clearPortNumMap();
- }
-
@Deactivate
protected void deactivate() {
+ osRouterService.removeListener(osRouterListener);
+ osNodeService.removeListener(osNodeListener);
+ instancePortService.removeListener(instancePortListener);
+ configService.unregisterProperties(getClass(), false);
packetService.removeProcessor(packetProcessor);
+ leadershipService.withdraw(appId.name());
eventExecutor.shutdown();
log.info("Stopped");
}
+ @Modified
+ protected void modified(ComponentContext context) {
+ Dictionary<?, ?> properties = context.getProperties();
+ Boolean flag;
+
+ flag = Tools.isPropertyEnabled(properties, USE_STATEFUL_SNAT);
+ if (flag == null) {
+ log.info("useStatefulSnat is not configured, " +
+ "using current value of {}", useStatefulSnat);
+ } else {
+ useStatefulSnat = flag;
+ log.info("Configured. useStatefulSnat is {}",
+ useStatefulSnat ? "enabled" : "disabled");
+ }
+
+ resetSnatRules();
+ }
+
private void processSnatPacket(PacketContext context, Ethernet eth) {
IPv4 iPacket = (IPv4) eth.getPayload();
InboundPacket packetIn = context.inPacket();
@@ -181,7 +273,7 @@
InstancePort srcInstPort = instancePortService.instancePort(eth.getSourceMAC());
if (srcInstPort == null) {
- log.error(ERR_PACKETIN + "source host(MAC:{}) does not exist",
+ log.error(ERR_PACKET_IN + "source host(MAC:{}) does not exist",
eth.getSourceMAC());
return;
}
@@ -233,7 +325,7 @@
if (osNet == null) {
final String error = String.format("%s network %s not found",
- ERR_PACKETIN, srcInstPort.networkId());
+ ERR_PACKET_IN, srcInstPort.networkId());
throw new IllegalStateException(error);
}
@@ -512,6 +604,605 @@
});
}
+ private void initializeUnusedPortNumSet() {
+ for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
+ if (!allocatedPortNumMap.containsKey(i)) {
+ unUsedPortNumSet.add(i);
+ }
+ }
+
+ clearPortNumMap();
+ }
+
+ private void resetSnatRules() {
+ if (useStatefulSnat) {
+ osRouterService.routerInterfaces().forEach(
+ routerIface -> {
+ setReactiveSnatRules(routerIface, false);
+ setStatefulSnatRules(routerIface, true);
+ }
+ );
+ } else {
+ osRouterService.routerInterfaces().forEach(
+ routerIface -> {
+ setStatefulSnatRules(routerIface, false);
+ setReactiveSnatRules(routerIface, true);
+ }
+ );
+ }
+ }
+
+ private void setRulesToGateway(OpenstackNode osNode,
+ String segmentId,
+ IpPrefix srcSubnet,
+ Type networkType,
+ boolean install) {
+ OpenstackNode sourceNatGateway =
+ osNodeService.completeNodes(GATEWAY).stream().findFirst().orElse(null);
+
+ if (sourceNatGateway == null) {
+ return;
+ }
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(srcSubnet.getIp4Prefix())
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+
+ switch (networkType) {
+ case VXLAN:
+ case GRE:
+ case GENEVE:
+ sBuilder.matchTunnelId(Long.parseLong(segmentId));
+ break;
+ case VLAN:
+ sBuilder.matchVlanId(VlanId.vlanId(segmentId));
+ break;
+ default:
+ final String error = String.format("%s %s",
+ ERR_UNSUPPORTED_NET_TYPE,
+ networkType.toString());
+ throw new IllegalStateException(error);
+ }
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ switch (networkType) {
+ case VXLAN:
+ case GRE:
+ case GENEVE:
+ PortNumber portNum = tunnelPortNumByNetType(networkType, osNode);
+ tBuilder.extension(buildExtension(
+ deviceService,
+ osNode.intgBridge(),
+ sourceNatGateway.dataIp().getIp4Address()),
+ osNode.intgBridge())
+ .setOutput(portNum);
+ break;
+
+ case VLAN:
+ tBuilder.setOutput(osNode.vlanPortNum());
+ break;
+
+ default:
+ break;
+ }
+
+ osFlowRuleService.setRule(
+ appId,
+ osNode.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_EXTERNAL_ROUTING_RULE,
+ ROUTING_TABLE,
+ install);
+ }
+
+
+ private void routerUpdated(Router osRouter) {
+ ExternalGateway exGateway = osRouter.getExternalGatewayInfo();
+
+ ExternalPeerRouter externalPeerRouter =
+ osNetworkAdminService.externalPeerRouter(exGateway);
+ VlanId vlanId = externalPeerRouter == null ? VlanId.NONE : externalPeerRouter.vlanId();
+
+ if (exGateway == null) {
+ deleteUnassociatedExternalPeerRouter();
+ osRouterService.routerInterfaces(osRouter.getId()).forEach(iface ->
+ setSourceNat(iface, false));
+ } else {
+ osNetworkAdminService.deriveExternalPeerRouterMac(exGateway, osRouter, vlanId);
+ osRouterService.routerInterfaces(osRouter.getId()).forEach(iface ->
+ setSourceNat(iface, exGateway.isEnableSnat()));
+ }
+ }
+
+ private void deleteUnassociatedExternalPeerRouter() {
+ log.trace("Deleting unassociated external peer router");
+
+ try {
+ Set<String> routerIps = Sets.newConcurrentHashSet();
+
+ osRouterService.routers().stream()
+ .filter(router -> getGatewayIpAddress(router) != null)
+ .map(router -> getGatewayIpAddress(router).toString())
+ .forEach(routerIps::add);
+
+ osNetworkAdminService.externalPeerRouters().stream()
+ .filter(externalPeerRouter ->
+ !routerIps.contains(externalPeerRouter.ipAddress().toString()))
+ .forEach(externalPeerRouter -> {
+ osNetworkAdminService
+ .deleteExternalPeerRouter(
+ externalPeerRouter.ipAddress().toString());
+ log.trace("Deleted unassociated external peer router {}",
+ externalPeerRouter.ipAddress().toString());
+ });
+ } catch (Exception e) {
+ log.error("Exception occurred because of {}", e.toString());
+ }
+ }
+
+ private void routerIfaceAdded(Router osRouter, RouterInterface osRouterIface) {
+ ExternalGateway exGateway = osRouter.getExternalGatewayInfo();
+ if (exGateway != null && exGateway.isEnableSnat()) {
+ setSourceNat(osRouterIface, true);
+ }
+ }
+
+ private void routerIfaceRemoved(Router osRouter, RouterInterface osRouterIface) {
+ ExternalGateway exGateway = osRouter.getExternalGatewayInfo();
+ if (exGateway != null && exGateway.isEnableSnat()) {
+ setSourceNat(osRouterIface, false);
+ }
+ }
+
+ private void setSourceNat(RouterInterface routerIface, boolean install) {
+ Subnet osSubnet = osNetworkAdminService.subnet(routerIface.getSubnetId());
+ Network osNet = osNetworkAdminService.network(osSubnet.getNetworkId());
+ Type netType = osNetworkAdminService.networkType(osSubnet.getNetworkId());
+
+ osNodeService.completeNodes(COMPUTE).forEach(cNode -> {
+ setRulesToGateway(cNode, osNet.getProviderSegID(),
+ IpPrefix.valueOf(osSubnet.getCidr()), netType, install);
+ });
+
+ if (useStatefulSnat) {
+ setStatefulSnatRules(routerIface, install);
+ } else {
+ setReactiveSnatRules(routerIface, install);
+ }
+
+ final String updateStr = install ? MSG_ENABLED : MSG_DISABLED;
+ log.info(updateStr + "external access for subnet({})", osSubnet.getCidr());
+ }
+
+ private void setStatefulSnatRules(RouterInterface routerIface, boolean install) {
+ Subnet osSubnet = osNetworkAdminService.subnet(routerIface.getSubnetId());
+ Network osNet = osNetworkAdminService.network(osSubnet.getNetworkId());
+ Type netType = osNetworkAdminService.networkType(osSubnet.getNetworkId());
+
+ if (netType == FLAT) {
+ return;
+ }
+
+ Optional<Router> osRouter = osRouterService.routers().stream()
+ .filter(router -> osRouterService.routerInterfaces(routerIface.getId()) != null)
+ .findAny();
+
+ if (!osRouter.isPresent()) {
+ log.error("Cannot find a router for router interface {} ", routerIface);
+ return;
+ }
+ IpAddress natAddress = getGatewayIpAddress(osRouter.get());
+ if (natAddress == null) {
+ return;
+ }
+ String netId = osNetworkAdminService.subnet(routerIface.getSubnetId()).getNetworkId();
+
+ osNodeService.completeNodes(GATEWAY)
+ .forEach(gwNode -> {
+ instancePortService.instancePorts(netId)
+ .stream()
+ .filter(port -> port.state() == ACTIVE)
+ .forEach(port -> setRulesForSnatIngressRule(gwNode.intgBridge(),
+ Long.parseLong(osNet.getProviderSegID()),
+ IpPrefix.valueOf(port.ipAddress(), VM_PREFIX),
+ port.deviceId(),
+ netType,
+ install));
+
+ setOvsNatIngressRule(gwNode.intgBridge(),
+ IpPrefix.valueOf(natAddress, VM_PREFIX),
+ Constants.DEFAULT_EXTERNAL_ROUTER_MAC, install);
+
+ if (gwNode.patchPortNum() != null) {
+ setOvsNatEgressRule(gwNode.intgBridge(),
+ natAddress, Long.parseLong(osNet.getProviderSegID()),
+ gwNode.patchPortNum(), install);
+ }
+ });
+ }
+
+ private IpAddress getGatewayIpAddress(Router osRouter) {
+
+ if (osRouter.getExternalGatewayInfo() == null) {
+ return null;
+ }
+ String extNetId = osNetworkAdminService.network(
+ osRouter.getExternalGatewayInfo().getNetworkId()).getId();
+ Optional<Subnet> extSubnet = osNetworkAdminService.subnets().stream()
+ .filter(subnet -> subnet.getNetworkId().equals(extNetId))
+ .findAny();
+
+ if (!extSubnet.isPresent()) {
+ log.error("Cannot find externel subnet for the router");
+ return null;
+ }
+
+ return IpAddress.valueOf(extSubnet.get().getGateway());
+ }
+
+ private void setReactiveSnatRules(RouterInterface routerIface, boolean install) {
+ Subnet osSubnet = osNetworkAdminService.subnet(routerIface.getSubnetId());
+ Network osNet = osNetworkAdminService.network(osSubnet.getNetworkId());
+ Type netType = osNetworkAdminService.networkType(osSubnet.getNetworkId());
+
+ osNodeService.completeNodes(GATEWAY)
+ .forEach(gwNode -> setRulesToController(
+ gwNode.intgBridge(),
+ osNet.getProviderSegID(),
+ IpPrefix.valueOf(osSubnet.getCidr()),
+ netType,
+ install));
+ }
+
+ private void setOvsNatIngressRule(DeviceId deviceId,
+ IpPrefix cidr,
+ MacAddress dstMac,
+ boolean install) {
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(cidr)
+ .build();
+
+ ExtensionTreatment natTreatment = RulePopulatorUtil
+ .niciraConnTrackTreatmentBuilder(driverService, deviceId)
+ .commit(false)
+ .natAction(true)
+ .table((short) 0)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setEthDst(dstMac)
+ .extension(natTreatment, deviceId)
+ .build();
+
+ osFlowRuleService.setRule(
+ appId,
+ deviceId,
+ selector,
+ treatment,
+ PRIORITY_STATEFUL_SNAT_RULE,
+ GW_COMMON_TABLE,
+ install);
+ }
+
+ private void setOvsNatEgressRule(DeviceId deviceId,
+ IpAddress natAddress,
+ long vni,
+ PortNumber output,
+ boolean install) {
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthDst(DEFAULT_GATEWAY_MAC)
+ .matchTunnelId(vni)
+ .build();
+
+ ExtensionTreatment natTreatment = RulePopulatorUtil
+ .niciraConnTrackTreatmentBuilder(driverService, deviceId)
+ .commit(true)
+ .natAction(true)
+ .natIp(natAddress)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .extension(natTreatment, deviceId)
+ .setEthDst(DEFAULT_EXTERNAL_ROUTER_MAC)
+ .setEthSrc(DEFAULT_GATEWAY_MAC)
+ .setOutput(output)
+ .build();
+
+ osFlowRuleService.setRule(
+ appId,
+ deviceId,
+ selector,
+ treatment,
+ PRIORITY_STATEFUL_SNAT_RULE,
+ GW_COMMON_TABLE,
+ install);
+ }
+
+ private void setRulesForSnatIngressRule(DeviceId deviceId,
+ Long vni,
+ IpPrefix destVmIp,
+ DeviceId dstDeviceId,
+ Type networkType,
+ boolean install) {
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(destVmIp)
+ .build();
+
+ PortNumber portNum = tunnelPortNumByNetType(networkType,
+ osNodeService.node(deviceId));
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setTunnelId(vni)
+ .extension(buildExtension(
+ deviceService,
+ deviceId,
+ osNodeService.node(dstDeviceId).dataIp().getIp4Address()),
+ deviceId)
+ .setOutput(portNum)
+ .build();
+
+ osFlowRuleService.setRule(
+ appId,
+ deviceId,
+ selector,
+ treatment,
+ PRIORITY_EXTERNAL_ROUTING_RULE,
+ Constants.GW_COMMON_TABLE,
+ install);
+ }
+
+ private void setRulesToController(DeviceId deviceId,
+ String segmentId,
+ IpPrefix srcSubnet,
+ Type networkType,
+ boolean install) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(srcSubnet)
+ .matchEthDst(Constants.DEFAULT_GATEWAY_MAC);
+
+ switch (networkType) {
+ case VXLAN:
+ case GRE:
+ case GENEVE:
+ sBuilder.matchTunnelId(Long.parseLong(segmentId));
+ break;
+ case VLAN:
+ sBuilder.matchVlanId(VlanId.vlanId(segmentId));
+ break;
+ default:
+ final String error = String.format("%s %s",
+ ERR_UNSUPPORTED_NET_TYPE,
+ networkType.toString());
+ throw new IllegalStateException(error);
+ }
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ if (networkType == VLAN) {
+ tBuilder.popVlan();
+ }
+
+ tBuilder.punt();
+
+ osFlowRuleService.setRule(
+ appId,
+ deviceId,
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_EXTERNAL_ROUTING_RULE,
+ GW_COMMON_TABLE,
+ install);
+ }
+
+ private class InternalInstancePortListener implements InstancePortListener {
+
+ private boolean isRelevantHelper(InstancePortEvent event) {
+ return mastershipService.isLocalMaster(event.subject().deviceId());
+ }
+
+ @Override
+ public void event(InstancePortEvent event) {
+ InstancePort instPort = event.subject();
+ switch (event.type()) {
+ case OPENSTACK_INSTANCE_PORT_DETECTED:
+ case OPENSTACK_INSTANCE_PORT_UPDATED:
+ eventExecutor.execute(() ->
+ processInstancePortDetection(event, instPort));
+ break;
+ case OPENSTACK_INSTANCE_PORT_VANISHED:
+ eventExecutor.execute(() ->
+ processInstancePortRemoval(event, instPort));
+ break;
+ case OPENSTACK_INSTANCE_MIGRATION_STARTED:
+ eventExecutor.execute(() ->
+ processInstanceMigrationStart(event, instPort));
+ break;
+ case OPENSTACK_INSTANCE_MIGRATION_ENDED:
+ eventExecutor.execute(() ->
+ processInstanceMigrationEnd(event, instPort));
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void processInstancePortDetection(InstancePortEvent event,
+ InstancePort instPort) {
+ if (!isRelevantHelper(event)) {
+ return;
+ }
+
+ log.info("RoutingHandler: Instance port detected MAC:{} IP:{}",
+ instPort.macAddress(),
+ instPort.ipAddress());
+
+ instPortDetected(event.subject());
+ }
+
+ private void processInstancePortRemoval(InstancePortEvent event,
+ InstancePort instPort) {
+ if (!isRelevantHelper(event)) {
+ return;
+ }
+
+ log.info("RoutingHandler: Instance port vanished MAC:{} IP:{}",
+ instPort.macAddress(),
+ instPort.ipAddress());
+
+ instPortRemoved(event.subject());
+ }
+
+ private void processInstanceMigrationStart(InstancePortEvent event,
+ InstancePort instPort) {
+ if (!isRelevantHelper(event)) {
+ return;
+ }
+
+ log.info("RoutingHandler: Migration started for MAC:{} IP:{}",
+ instPort.macAddress(),
+ instPort.ipAddress());
+
+ instPortDetected(instPort);
+ }
+
+ private void processInstanceMigrationEnd(InstancePortEvent event,
+ InstancePort instPort) {
+ log.info("RoutingHandler: Migration finished for MAC:{} IP:{}",
+ instPort.macAddress(),
+ instPort.ipAddress());
+ // TODO: need to reconfigure rules to point to update VM
+ }
+
+ private void instPortDetected(InstancePort instPort) {
+ Network network = osNetworkAdminService.network(instPort.networkId());
+ Type netType = osNetworkAdminService.networkType(instPort.networkId());
+
+ if (netType == FLAT) {
+ return;
+ }
+
+ if (useStatefulSnat) {
+ osNodeService.completeNodes(GATEWAY)
+ .forEach(gwNode -> setRulesForSnatIngressRule(
+ gwNode.intgBridge(),
+ Long.parseLong(network.getProviderSegID()),
+ IpPrefix.valueOf(instPort.ipAddress(), VM_PREFIX),
+ instPort.deviceId(), netType, true));
+ }
+ }
+
+ private void instPortRemoved(InstancePort instPort) {
+ Network network = osNetworkAdminService.network(instPort.networkId());
+ Type netType = osNetworkAdminService.networkType(instPort.networkId());
+
+ if (netType == FLAT) {
+ return;
+ }
+
+ if (useStatefulSnat) {
+ osNodeService.completeNodes(GATEWAY)
+ .forEach(gwNode -> setRulesForSnatIngressRule(
+ gwNode.intgBridge(),
+ Long.parseLong(network.getProviderSegID()),
+ IpPrefix.valueOf(instPort.ipAddress(), VM_PREFIX),
+ instPort.deviceId(), netType, false));
+ }
+ }
+ }
+
+ private class InternalRouterEventListener implements OpenstackRouterListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(OpenstackRouterEvent event) {
+ switch (event.type()) {
+ case OPENSTACK_ROUTER_CREATED:
+ eventExecutor.execute(() -> processRouterCreation(event));
+ break;
+ case OPENSTACK_ROUTER_UPDATED:
+ eventExecutor.execute(() -> processRouterUpdate(event));
+ break;
+ case OPENSTACK_ROUTER_INTERFACE_ADDED:
+ eventExecutor.execute(() -> processRouterIntfCreation(event));
+ break;
+ case OPENSTACK_ROUTER_INTERFACE_REMOVED:
+ eventExecutor.execute(() -> processRouterIntfRemoval(event));
+ break;
+ case OPENSTACK_ROUTER_GATEWAY_ADDED:
+ log.debug("Router external gateway {} added",
+ event.externalGateway().getNetworkId());
+ break;
+ case OPENSTACK_ROUTER_GATEWAY_REMOVED:
+ log.debug("Router external gateway {} removed",
+ event.externalGateway().getNetworkId());
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void processRouterCreation(OpenstackRouterEvent event) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ log.debug("Router(name:{}, ID:{}) is created",
+ event.subject().getName(),
+ event.subject().getId());
+
+ routerUpdated(event.subject());
+ }
+
+ private void processRouterUpdate(OpenstackRouterEvent event) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ log.debug("Router(name:{}, ID:{}) is updated",
+ event.subject().getName(),
+ event.subject().getId());
+
+ routerUpdated(event.subject());
+ }
+
+ private void processRouterIntfCreation(OpenstackRouterEvent event) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ log.debug("Router interface {} added to router {}",
+ event.routerIface().getPortId(),
+ event.routerIface().getId());
+
+ routerIfaceAdded(event.subject(), event.routerIface());
+ }
+
+ private void processRouterIntfRemoval(OpenstackRouterEvent event) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ log.debug("Router interface {} removed from router {}",
+ event.routerIface().getPortId(),
+ event.routerIface().getId());
+
+ routerIfaceRemoved(event.subject(), event.routerIface());
+ }
+ }
+
private class InternalPacketProcessor implements PacketProcessor {
@Override
@@ -556,4 +1247,42 @@
return gateways.contains(context.inPacket().receivedFrom().deviceId());
}
}
+
+ private class InternalNodeEventListener implements OpenstackNodeListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(OpenstackNodeEvent event) {
+ OpenstackNode osNode = event.subject();
+ switch (event.type()) {
+ case OPENSTACK_NODE_COMPLETE:
+ case OPENSTACK_NODE_INCOMPLETE:
+ case OPENSTACK_NODE_UPDATED:
+ case OPENSTACK_NODE_REMOVED:
+ eventExecutor.execute(() -> {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ reconfigureRouters(osNode);
+ });
+ break;
+ case OPENSTACK_NODE_CREATED:
+ default:
+ break;
+ }
+ }
+
+ private void reconfigureRouters(OpenstackNode osNode) {
+ osRouterService.routers().forEach(osRouter -> {
+ routerUpdated(osRouter);
+ osRouterService.routerInterfaces(osRouter.getId()).forEach(iface -> {
+ routerIfaceAdded(osRouter, iface);
+ });
+ });
+ log.info("Reconfigure routers for {}", osNode.hostname());
+ }
+ }
}