Placed API and implementation into separate packages
Change-Id: If8a9223a7a225db1b2aa2d09738857af482736bc
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtn.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtn.java
new file mode 100644
index 0000000..967ca02
--- /dev/null
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtn.java
@@ -0,0 +1,806 @@
+/*
+ * Copyright 2015-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cordvtn.impl;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.VlanId;
+import org.onosproject.cordvtn.api.CordService;
+import org.onosproject.cordvtn.api.CordServiceId;
+import org.onosproject.cordvtn.api.CordVtnConfig;
+import org.onosproject.cordvtn.api.CordVtnNode;
+import org.onosproject.cordvtn.api.CordVtnService;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.dhcp.DhcpService;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.net.Host;
+import org.onosproject.net.HostId;
+import org.onosproject.net.HostLocation;
+import org.onosproject.net.Port;
+import org.onosproject.net.config.ConfigFactory;
+import org.onosproject.net.config.NetworkConfigEvent;
+import org.onosproject.net.config.NetworkConfigListener;
+import org.onosproject.net.config.NetworkConfigRegistry;
+import org.onosproject.net.config.NetworkConfigService;
+import org.onosproject.net.config.basics.SubjectFactories;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.FlowRuleService;
+import org.onosproject.net.group.GroupService;
+import org.onosproject.net.host.DefaultHostDescription;
+import org.onosproject.net.host.HostDescription;
+import org.onosproject.net.host.HostEvent;
+import org.onosproject.net.host.HostListener;
+import org.onosproject.net.host.HostProvider;
+import org.onosproject.net.host.HostProviderRegistry;
+import org.onosproject.net.host.HostProviderService;
+import org.onosproject.net.host.HostService;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketProcessor;
+import org.onosproject.net.packet.PacketService;
+import org.onosproject.net.provider.AbstractProvider;
+import org.onosproject.net.provider.ProviderId;
+import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackNetwork;
+import org.onosproject.openstackinterface.OpenstackPort;
+import org.onosproject.openstackinterface.OpenstackSubnet;
+import org.slf4j.Logger;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Provisions virtual tenant networks with service chaining capability
+ * in OpenStack environment.
+ */
+@Component(immediate = true)
+@Service
+public class CordVtn extends AbstractProvider implements CordVtnService, HostProvider {
+
+ protected final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected NetworkConfigRegistry configRegistry;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected NetworkConfigService configService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected HostProviderRegistry hostProviderRegistry;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected HostService hostService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DriverService driverService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected FlowRuleService flowRuleService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected PacketService packetService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected GroupService groupService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackInterfaceService openstackService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DhcpService dhcpService;
+
+ private final ConfigFactory configFactory =
+ new ConfigFactory(SubjectFactories.APP_SUBJECT_FACTORY, CordVtnConfig.class, "cordvtn") {
+ @Override
+ public CordVtnConfig createConfig() {
+ return new CordVtnConfig();
+ }
+ };
+
+ private static final String DEFAULT_TUNNEL = "vxlan";
+ private static final String SERVICE_ID = "serviceId";
+ private static final String OPENSTACK_PORT_ID = "openstackPortId";
+ private static final String DATA_PLANE_IP = "dataPlaneIp";
+ private static final String DATA_PLANE_INTF = "dataPlaneIntf";
+ private static final String S_TAG = "stag";
+ private static final String VSG_HOST_ID = "vsgHostId";
+ private static final String CREATED_TIME = "createdTime";
+
+ private static final Ip4Address DEFAULT_DNS = Ip4Address.valueOf("8.8.8.8");
+
+ private final ExecutorService eventExecutor =
+ newSingleThreadScheduledExecutor(groupedThreads("onos/cordvtn", "event-handler"));
+
+ private final PacketProcessor packetProcessor = new InternalPacketProcessor();
+ private final HostListener hostListener = new InternalHostListener();
+ private final NetworkConfigListener configListener = new InternalConfigListener();
+
+ private ApplicationId appId;
+ private HostProviderService hostProvider;
+ private CordVtnRuleInstaller ruleInstaller;
+ private CordVtnArpProxy arpProxy;
+ private volatile MacAddress privateGatewayMac = MacAddress.NONE;
+
+ /**
+ * Creates an cordvtn host location provider.
+ */
+ public CordVtn() {
+ super(new ProviderId("host", CORDVTN_APP_ID));
+ }
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication("org.onosproject.cordvtn");
+ ruleInstaller = new CordVtnRuleInstaller(appId, flowRuleService,
+ deviceService,
+ driverService,
+ groupService,
+ configRegistry,
+ DEFAULT_TUNNEL);
+
+ arpProxy = new CordVtnArpProxy(appId, packetService, hostService);
+ packetService.addProcessor(packetProcessor, PacketProcessor.director(0));
+ arpProxy.requestPacket();
+
+ hostService.addListener(hostListener);
+ hostProvider = hostProviderRegistry.register(this);
+
+ configRegistry.registerConfigFactory(configFactory);
+ configService.addListener(configListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ hostProviderRegistry.unregister(this);
+ hostService.removeListener(hostListener);
+
+ packetService.removeProcessor(packetProcessor);
+
+ configRegistry.unregisterConfigFactory(configFactory);
+ configService.removeListener(configListener);
+
+ eventExecutor.shutdown();
+ log.info("Stopped");
+ }
+
+ @Override
+ public void triggerProbe(Host host) {
+ /*
+ * Note: In CORD deployment, we assume that all hosts are configured.
+ * Therefore no probe is required.
+ */
+ }
+
+ @Override
+ public void createServiceDependency(CordServiceId tServiceId, CordServiceId pServiceId,
+ boolean isBidirectional) {
+ CordService tService = getCordService(tServiceId);
+ CordService pService = getCordService(pServiceId);
+
+ if (tService == null || pService == null) {
+ log.error("Failed to create CordService for {}", tServiceId.id());
+ return;
+ }
+
+ log.info("Service dependency from {} to {} created.", tService.id().id(), pService.id().id());
+ ruleInstaller.populateServiceDependencyRules(tService, pService, isBidirectional);
+ }
+
+ @Override
+ public void removeServiceDependency(CordServiceId tServiceId, CordServiceId pServiceId) {
+ CordService tService = getCordService(tServiceId);
+ CordService pService = getCordService(pServiceId);
+
+ if (tService == null || pService == null) {
+ log.error("Failed to create CordService for {}", tServiceId.id());
+ return;
+ }
+
+ log.info("Service dependency from {} to {} removed.", tService.id().id(), pService.id().id());
+ ruleInstaller.removeServiceDependencyRules(tService, pService);
+ }
+
+ @Override
+ public void addServiceVm(CordVtnNode node, ConnectPoint connectPoint) {
+ Port port = deviceService.getPort(connectPoint.deviceId(), connectPoint.port());
+ OpenstackPort vPort = openstackService.port(port);
+ if (vPort == null) {
+ log.warn("Failed to get OpenstackPort for {}", getPortName(port));
+ return;
+ }
+
+ MacAddress mac = vPort.macAddress();
+ HostId hostId = HostId.hostId(mac);
+
+ Host existingHost = hostService.getHost(hostId);
+ if (existingHost != null) {
+ String serviceId = existingHost.annotations().value(SERVICE_ID);
+ if (serviceId == null || !serviceId.equals(vPort.networkId())) {
+ // this host is not injected by cordvtn or a stale host, remove it
+ hostProvider.hostVanished(existingHost.id());
+ }
+ }
+
+ // Included CREATED_TIME to annotation intentionally to trigger HOST_UPDATED
+ // event so that the flow rule population for this host can happen.
+ // This ensures refreshing data plane by pushing network config always make
+ // the data plane synced.
+ Set<IpAddress> fixedIp = Sets.newHashSet(vPort.fixedIps().values());
+ DefaultAnnotations.Builder annotations = DefaultAnnotations.builder()
+ .set(SERVICE_ID, vPort.networkId())
+ .set(OPENSTACK_PORT_ID, vPort.id())
+ .set(DATA_PLANE_IP, node.dpIp().ip().toString())
+ .set(DATA_PLANE_INTF, node.dpIntf())
+ .set(CREATED_TIME, String.valueOf(System.currentTimeMillis()));
+
+ String serviceVlan = getServiceVlan(vPort);
+ if (serviceVlan != null) {
+ annotations.set(S_TAG, serviceVlan);
+ }
+
+ HostDescription hostDesc = new DefaultHostDescription(
+ mac,
+ VlanId.NONE,
+ new HostLocation(connectPoint, System.currentTimeMillis()),
+ fixedIp,
+ annotations.build());
+
+ hostProvider.hostDetected(hostId, hostDesc, false);
+ }
+
+ @Override
+ public void removeServiceVm(ConnectPoint connectPoint) {
+ hostService.getConnectedHosts(connectPoint)
+ .stream()
+ .forEach(host -> hostProvider.hostVanished(host.id()));
+ }
+
+ @Override
+ public void updateVirtualSubscriberGateways(HostId vSgHostId, String serviceVlan,
+ Map<IpAddress, MacAddress> vSgs) {
+ Host vSgHost = hostService.getHost(vSgHostId);
+ if (vSgHost == null || !vSgHost.annotations().value(S_TAG).equals(serviceVlan)) {
+ log.debug("Invalid vSG updates for {}", serviceVlan);
+ return;
+ }
+
+ log.info("Updates vSGs in {} with {}", vSgHost.id(), vSgs.toString());
+ vSgs.entrySet().stream()
+ .filter(entry -> hostService.getHostsByMac(entry.getValue()).isEmpty())
+ .forEach(entry -> addVirtualSubscriberGateway(
+ vSgHost,
+ entry.getKey(),
+ entry.getValue(),
+ serviceVlan));
+
+ hostService.getConnectedHosts(vSgHost.location()).stream()
+ .filter(host -> !host.mac().equals(vSgHost.mac()))
+ .filter(host -> !vSgs.values().contains(host.mac()))
+ .forEach(host -> {
+ log.info("Removed vSG {}", host.toString());
+ hostProvider.hostVanished(host.id());
+ });
+ }
+
+ /**
+ * Adds virtual subscriber gateway to the system.
+ *
+ * @param vSgHost host virtual machine of this vSG
+ * @param vSgIp vSG ip address
+ * @param vSgMac vSG mac address
+ * @param serviceVlan service vlan
+ */
+ private void addVirtualSubscriberGateway(Host vSgHost, IpAddress vSgIp, MacAddress vSgMac,
+ String serviceVlan) {
+ log.info("vSG with IP({}) MAC({}) added", vSgIp.toString(), vSgMac.toString());
+
+ HostId hostId = HostId.hostId(vSgMac);
+ DefaultAnnotations.Builder annotations = DefaultAnnotations.builder()
+ .set(S_TAG, serviceVlan)
+ .set(VSG_HOST_ID, vSgHost.id().toString())
+ .set(CREATED_TIME, String.valueOf(System.currentTimeMillis()));
+
+ HostDescription hostDesc = new DefaultHostDescription(
+ vSgMac,
+ VlanId.NONE,
+ vSgHost.location(),
+ Sets.newHashSet(vSgIp),
+ annotations.build());
+
+ hostProvider.hostDetected(hostId, hostDesc, false);
+ }
+
+ /**
+ * Returns public ip addresses of vSGs running inside a give vSG host.
+ *
+ * @param vSgHost vSG host
+ * @return map of ip and mac address, or empty map
+ */
+ private Map<IpAddress, MacAddress> getSubscriberGateways(Host vSgHost) {
+ String vPortId = vSgHost.annotations().value(OPENSTACK_PORT_ID);
+ String serviceVlan = vSgHost.annotations().value(S_TAG);
+
+ OpenstackPort vPort = openstackService.port(vPortId);
+ if (vPort == null) {
+ log.warn("Failed to get OpenStack port {} for VM {}", vPortId, vSgHost.id());
+ return Maps.newHashMap();
+ }
+
+ if (!serviceVlan.equals(getServiceVlan(vPort))) {
+ log.error("Host({}) s-tag does not match with vPort s-tag", vSgHost.id());
+ return Maps.newHashMap();
+ }
+
+ return vPort.allowedAddressPairs();
+ }
+
+ /**
+ * Returns CordService by service ID.
+ *
+ * @param serviceId service id
+ * @return cord service, or null if it fails to get network from OpenStack
+ */
+ private CordService getCordService(CordServiceId serviceId) {
+ OpenstackNetwork vNet = openstackService.network(serviceId.id());
+ if (vNet == null) {
+ log.warn("Couldn't find OpenStack network for service {}", serviceId.id());
+ return null;
+ }
+
+ OpenstackSubnet subnet = vNet.subnets().stream()
+ .findFirst()
+ .orElse(null);
+ if (subnet == null) {
+ log.warn("Couldn't find OpenStack subnet for service {}", serviceId.id());
+ return null;
+ }
+
+ Set<CordServiceId> tServices = Sets.newHashSet();
+ // TODO get tenant services from XOS
+
+ Map<Host, IpAddress> hosts = getHostsWithOpenstackNetwork(vNet)
+ .stream()
+ .collect(Collectors.toMap(host -> host, this::getTunnelIp));
+
+ return new CordService(vNet, subnet, hosts, tServices);
+ }
+
+ /**
+ * Returns CordService by OpenStack network.
+ *
+ * @param vNet OpenStack network
+ * @return cord service
+ */
+ private CordService getCordService(OpenstackNetwork vNet) {
+ checkNotNull(vNet);
+
+ CordServiceId serviceId = CordServiceId.of(vNet.id());
+ OpenstackSubnet subnet = vNet.subnets().stream()
+ .findFirst()
+ .orElse(null);
+ if (subnet == null) {
+ log.warn("Couldn't find OpenStack subnet for service {}", serviceId);
+ return null;
+ }
+
+ Set<CordServiceId> tServices = Sets.newHashSet();
+ // TODO get tenant services from XOS
+
+ Map<Host, IpAddress> hosts = getHostsWithOpenstackNetwork(vNet)
+ .stream()
+ .collect(Collectors.toMap(host -> host, this::getTunnelIp));
+
+ return new CordService(vNet, subnet, hosts, tServices);
+ }
+
+ /**
+ * Returns IP address for tunneling for a given host.
+ *
+ * @param host host
+ * @return ip address, or null
+ */
+ private IpAddress getTunnelIp(Host host) {
+ String ip = host.annotations().value(DATA_PLANE_IP);
+ return ip == null ? null : IpAddress.valueOf(ip);
+ }
+
+ /**
+ * Returns port name.
+ *
+ * @param port port
+ * @return port name
+ */
+ private String getPortName(Port port) {
+ return port.annotations().value("portName");
+ }
+
+ /**
+ * Returns s-tag from a given OpenStack port.
+ *
+ * @param vPort openstack port
+ * @return s-tag string
+ */
+ private String getServiceVlan(OpenstackPort vPort) {
+ checkNotNull(vPort);
+
+ if (vPort.name() != null && vPort.name().startsWith(S_TAG)) {
+ return vPort.name().split("-")[1];
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns service ID of this host.
+ *
+ * @param host host
+ * @return service id, or null if not found
+ */
+ private String getServiceId(Host host) {
+ return host.annotations().value(SERVICE_ID);
+ }
+
+ /**
+ * Returns hosts associated with a given OpenStack network.
+ *
+ * @param vNet openstack network
+ * @return set of hosts
+ */
+ private Set<Host> getHostsWithOpenstackNetwork(OpenstackNetwork vNet) {
+ checkNotNull(vNet);
+
+ String vNetId = vNet.id();
+ return StreamSupport.stream(hostService.getHosts().spliterator(), false)
+ .filter(host -> Objects.equals(vNetId, getServiceId(host)))
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * Registers static DHCP lease for a given host.
+ *
+ * @param host host
+ * @param service cord service
+ */
+ private void registerDhcpLease(Host host, CordService service) {
+ List<Ip4Address> options = Lists.newArrayList();
+ options.add(Ip4Address.makeMaskPrefix(service.serviceIpRange().prefixLength()));
+ options.add(service.serviceIp().getIp4Address());
+ options.add(service.serviceIp().getIp4Address());
+ options.add(DEFAULT_DNS);
+
+ log.debug("Set static DHCP mapping for {}", host.mac());
+ dhcpService.setStaticMapping(host.mac(),
+ host.ipAddresses().stream().findFirst().get().getIp4Address(),
+ true,
+ options);
+ }
+
+ /**
+ * Handles VM detected situation.
+ *
+ * @param host host
+ */
+ private void serviceVmAdded(Host host) {
+ String serviceVlan = host.annotations().value(S_TAG);
+ if (serviceVlan != null) {
+ virtualSubscriberGatewayAdded(host, serviceVlan);
+ }
+
+ String vNetId = host.annotations().value(SERVICE_ID);
+ if (vNetId == null) {
+ // ignore this host, it is not the service VM, or it's a vSG
+ return;
+ }
+
+ OpenstackNetwork vNet = openstackService.network(vNetId);
+ if (vNet == null) {
+ log.warn("Failed to get OpenStack network {} for VM {}.",
+ vNetId, host.id());
+ return;
+ }
+
+ log.info("VM is detected, MAC: {} IP: {}",
+ host.mac(),
+ host.ipAddresses().stream().findFirst().get());
+
+ CordService service = getCordService(vNet);
+ if (service == null) {
+ return;
+ }
+
+ switch (service.serviceType()) {
+ case MANAGEMENT:
+ ruleInstaller.populateManagementNetworkRules(host, service);
+ break;
+ case PRIVATE:
+ arpProxy.addGateway(service.serviceIp(), privateGatewayMac);
+ case PUBLIC:
+ default:
+ // TODO check if the service needs an update on its group buckets after done CORD-433
+ ruleInstaller.updateServiceGroup(service);
+ // sends gratuitous ARP here for the case of adding existing VMs
+ // when ONOS or cordvtn app is restarted
+ arpProxy.sendGratuitousArpForGateway(service.serviceIp(), Sets.newHashSet(host));
+ break;
+ }
+
+ registerDhcpLease(host, service);
+ ruleInstaller.populateBasicConnectionRules(host, getTunnelIp(host), vNet);
+ }
+
+ /**
+ * Handles VM removed situation.
+ *
+ * @param host host
+ */
+ private void serviceVmRemoved(Host host) {
+ String serviceVlan = host.annotations().value(S_TAG);
+ if (serviceVlan != null) {
+ virtualSubscriberGatewayRemoved(host);
+ }
+
+ String vNetId = host.annotations().value(SERVICE_ID);
+ if (vNetId == null) {
+ // ignore it, it's not the service VM or it's a vSG
+ return;
+ }
+
+ OpenstackNetwork vNet = openstackService.network(vNetId);
+ if (vNet == null) {
+ log.warn("Failed to get OpenStack network {} for VM {}",
+ vNetId, host.id());
+ return;
+ }
+
+ log.info("VM is vanished, MAC: {} IP: {}",
+ host.mac(),
+ host.ipAddresses().stream().findFirst().get());
+
+ ruleInstaller.removeBasicConnectionRules(host);
+ dhcpService.removeStaticMapping(host.mac());
+
+ CordService service = getCordService(vNet);
+ if (service == null) {
+ return;
+ }
+
+ switch (service.serviceType()) {
+ case MANAGEMENT:
+ ruleInstaller.removeManagementNetworkRules(host, service);
+ break;
+ case PRIVATE:
+ if (getHostsWithOpenstackNetwork(vNet).isEmpty()) {
+ arpProxy.removeGateway(service.serviceIp());
+ }
+ case PUBLIC:
+ default:
+ // TODO check if the service needs an update on its group buckets after done CORD-433
+ ruleInstaller.updateServiceGroup(service);
+ break;
+ }
+ }
+
+
+ /**
+ * Handles virtual subscriber gateway VM or container.
+ *
+ * @param host new host with stag, it can be vsg VM or vsg
+ * @param serviceVlan service vlan
+ */
+ private void virtualSubscriberGatewayAdded(Host host, String serviceVlan) {
+ Map<IpAddress, MacAddress> vSgs;
+ Host vSgHost;
+
+ String vSgHostId = host.annotations().value(VSG_HOST_ID);
+ if (vSgHostId == null) {
+ log.debug("vSG VM detected {}", host.id());
+
+ vSgHost = host;
+ vSgs = getSubscriberGateways(vSgHost);
+ vSgs.entrySet().stream().forEach(entry -> addVirtualSubscriberGateway(
+ vSgHost,
+ entry.getKey(),
+ entry.getValue(),
+ serviceVlan));
+ } else {
+ vSgHost = hostService.getHost(HostId.hostId(vSgHostId));
+ if (vSgHost == null) {
+ return;
+ }
+
+ log.debug("vSG detected {}", host.id());
+ vSgs = getSubscriberGateways(vSgHost);
+ }
+
+ ruleInstaller.populateSubscriberGatewayRules(vSgHost, vSgs.keySet());
+ }
+
+ /**
+ * Handles virtual subscriber gateway removed.
+ *
+ * @param vSg vsg host to remove
+ */
+ private void virtualSubscriberGatewayRemoved(Host vSg) {
+ String vSgHostId = vSg.annotations().value(VSG_HOST_ID);
+ if (vSgHostId == null) {
+ return;
+ }
+
+ Host vSgHost = hostService.getHost(HostId.hostId(vSgHostId));
+ if (vSgHost == null) {
+ return;
+ }
+
+ log.info("vSG removed {}", vSg.id());
+ Map<IpAddress, MacAddress> vSgs = getSubscriberGateways(vSgHost);
+ ruleInstaller.populateSubscriberGatewayRules(vSgHost, vSgs.keySet());
+ }
+
+ /**
+ * Sets service network gateway MAC address and sends out gratuitous ARP to all
+ * VMs to update the gateway MAC address.
+ *
+ * @param newMac mac address to update
+ */
+ private void setPrivateGatewayMac(MacAddress newMac) {
+ if (newMac == null || newMac.equals(privateGatewayMac)) {
+ // no updates, do nothing
+ return;
+ }
+
+ privateGatewayMac = newMac;
+ log.debug("Set service gateway MAC address to {}", privateGatewayMac.toString());
+
+ // TODO get existing service list from XOS and replace the loop below
+ Set<String> vNets = Sets.newHashSet();
+ hostService.getHosts().forEach(host -> vNets.add(host.annotations().value(SERVICE_ID)));
+ vNets.remove(null);
+
+ vNets.stream().forEach(vNet -> {
+ CordService service = getCordService(CordServiceId.of(vNet));
+ if (service != null) {
+ arpProxy.addGateway(service.serviceIp(), privateGatewayMac);
+ arpProxy.sendGratuitousArpForGateway(service.serviceIp(), service.hosts().keySet());
+ }
+ });
+ }
+
+ /**
+ * Sets public gateway MAC address.
+ *
+ * @param publicGateways gateway ip and mac address pairs
+ */
+ private void setPublicGatewayMac(Map<IpAddress, MacAddress> publicGateways) {
+ publicGateways.entrySet()
+ .stream()
+ .forEach(entry -> {
+ arpProxy.addGateway(entry.getKey(), entry.getValue());
+ log.info("Added public gateway IP {}, MAC {}",
+ entry.getKey().toString(), entry.getValue().toString());
+ });
+ // TODO notice gateway MAC change to VMs holds this gateway IP
+ }
+
+ /**
+ * Updates configurations.
+ */
+ private void readConfiguration() {
+ CordVtnConfig config = configRegistry.getConfig(appId, CordVtnConfig.class);
+ if (config == null) {
+ log.debug("No configuration found");
+ return;
+ }
+
+ setPrivateGatewayMac(config.privateGatewayMac());
+ setPublicGatewayMac(config.publicGateways());
+ }
+
+ private class InternalHostListener implements HostListener {
+
+ @Override
+ public void event(HostEvent event) {
+ Host host = event.subject();
+ if (!mastershipService.isLocalMaster(host.location().deviceId())) {
+ // do not allow to proceed without mastership
+ return;
+ }
+
+ switch (event.type()) {
+ case HOST_UPDATED:
+ case HOST_ADDED:
+ eventExecutor.submit(() -> serviceVmAdded(host));
+ break;
+ case HOST_REMOVED:
+ eventExecutor.submit(() -> serviceVmRemoved(host));
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ private class InternalPacketProcessor implements PacketProcessor {
+
+ @Override
+ public void process(PacketContext context) {
+ if (context.isHandled()) {
+ return;
+ }
+
+ Ethernet ethPacket = context.inPacket().parsed();
+ if (ethPacket == null || ethPacket.getEtherType() != Ethernet.TYPE_ARP) {
+ return;
+ }
+
+ arpProxy.processArpPacket(context, ethPacket);
+ }
+ }
+
+ private class InternalConfigListener implements NetworkConfigListener {
+
+ @Override
+ public void event(NetworkConfigEvent event) {
+ if (!event.configClass().equals(CordVtnConfig.class)) {
+ return;
+ }
+
+ switch (event.type()) {
+ case CONFIG_ADDED:
+ case CONFIG_UPDATED:
+ log.info("Network configuration changed");
+ eventExecutor.execute(CordVtn.this::readConfiguration);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnArpProxy.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnArpProxy.java
new file mode 100644
index 0000000..c0b1d61
--- /dev/null
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnArpProxy.java
@@ -0,0 +1,243 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cordvtn.impl;
+
+import com.google.common.collect.Maps;
+import org.onlab.packet.ARP;
+import org.onlab.packet.EthType;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.net.Host;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.host.HostService;
+import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketPriority;
+import org.onosproject.net.packet.PacketService;
+import org.slf4j.Logger;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Handles ARP requests for virtual network service IPs.
+ */
+public class CordVtnArpProxy {
+ protected final Logger log = getLogger(getClass());
+
+ private final ApplicationId appId;
+ private final PacketService packetService;
+ private final HostService hostService;
+
+ private final Map<Ip4Address, MacAddress> gateways = Maps.newConcurrentMap();
+
+ /**
+ * Default constructor.
+ *
+ * @param appId application id
+ * @param packetService packet service
+ * @param hostService host service reference
+ */
+ public CordVtnArpProxy(ApplicationId appId, PacketService packetService, HostService hostService) {
+ this.appId = appId;
+ this.packetService = packetService;
+ this.hostService = hostService;
+ }
+
+ /**
+ * Requests ARP packet.
+ */
+ public void requestPacket() {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .build();
+
+ packetService.requestPackets(selector,
+ PacketPriority.CONTROL,
+ appId,
+ Optional.empty());
+ }
+
+ /**
+ * Cancels ARP packet.
+ */
+ public void cancelPacket() {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .build();
+
+ packetService.cancelPackets(selector,
+ PacketPriority.CONTROL,
+ appId,
+ Optional.empty());
+ }
+
+ /**
+ * Adds a given gateway IP and MAC address to this ARP proxy.
+ *
+ * @param gatewayIp gateway ip address
+ * @param gatewayMac gateway mac address
+ */
+ public void addGateway(IpAddress gatewayIp, MacAddress gatewayMac) {
+ checkNotNull(gatewayIp);
+ checkNotNull(gatewayMac);
+ gateways.put(gatewayIp.getIp4Address(), gatewayMac);
+ }
+
+ /**
+ * Removes a given service IP address from this ARP proxy.
+ *
+ * @param gatewayIp gateway ip address
+ */
+ public void removeGateway(IpAddress gatewayIp) {
+ checkNotNull(gatewayIp);
+ gateways.remove(gatewayIp.getIp4Address());
+ }
+
+ /**
+ * Emits ARP reply with fake MAC address for a given ARP request.
+ * It only handles requests for the registered service IPs, and the other
+ * requests can be handled by other ARP handlers like openstackSwitching or
+ * proxyArp, for example.
+ *
+ * @param context packet context
+ * @param ethPacket ethernet packet
+ */
+ public void processArpPacket(PacketContext context, Ethernet ethPacket) {
+ ARP arpPacket = (ARP) ethPacket.getPayload();
+ if (arpPacket.getOpCode() != ARP.OP_REQUEST) {
+ return;
+ }
+
+ Ip4Address targetIp = Ip4Address.valueOf(arpPacket.getTargetProtocolAddress());
+
+ MacAddress gatewayMac = gateways.get(targetIp);
+ MacAddress replyMac = gatewayMac != null ? gatewayMac : getMacFromHostService(targetIp);
+
+ if (replyMac.equals(MacAddress.NONE)) {
+ log.debug("Failed to find MAC for {}", targetIp.toString());
+ context.block();
+ return;
+ }
+
+ log.trace("Send ARP reply for {} with {}", targetIp.toString(), replyMac.toString());
+ Ethernet ethReply = ARP.buildArpReply(
+ targetIp,
+ replyMac,
+ ethPacket);
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(context.inPacket().receivedFrom().port())
+ .build();
+
+ packetService.emit(new DefaultOutboundPacket(
+ context.inPacket().receivedFrom().deviceId(),
+ treatment,
+ ByteBuffer.wrap(ethReply.serialize())));
+
+ context.block();
+ }
+
+ /**
+ * Emits gratuitous ARP when a gateway mac address has been changed.
+ *
+ * @param gatewayIp gateway ip address to update MAC
+ * @param hosts set of hosts to send gratuitous ARP packet
+ */
+ public void sendGratuitousArpForGateway(IpAddress gatewayIp, Set<Host> hosts) {
+ MacAddress gatewayMac = gateways.get(gatewayIp.getIp4Address());
+ if (gatewayMac == null) {
+ log.debug("Gateway {} is not registered to ARP proxy", gatewayIp.toString());
+ return;
+ }
+
+ Ethernet ethArp = buildGratuitousArp(gatewayIp.getIp4Address(), gatewayMac);
+ hosts.stream().forEach(host -> {
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(host.location().port())
+ .build();
+
+ packetService.emit(new DefaultOutboundPacket(
+ host.location().deviceId(),
+ treatment,
+ ByteBuffer.wrap(ethArp.serialize())));
+ });
+ }
+
+ /**
+ * Builds gratuitous ARP packet with a given IP and MAC address.
+ *
+ * @param ip ip address for TPA and SPA
+ * @param mac new mac address
+ * @return ethernet packet
+ */
+ private Ethernet buildGratuitousArp(IpAddress ip, MacAddress mac) {
+ Ethernet eth = new Ethernet();
+
+ eth.setEtherType(Ethernet.TYPE_ARP);
+ eth.setSourceMACAddress(mac);
+ eth.setDestinationMACAddress(MacAddress.BROADCAST);
+
+ ARP arp = new ARP();
+ arp.setOpCode(ARP.OP_REQUEST);
+ arp.setHardwareType(ARP.HW_TYPE_ETHERNET);
+ arp.setHardwareAddressLength((byte) Ethernet.DATALAYER_ADDRESS_LENGTH);
+ arp.setProtocolType(ARP.PROTO_TYPE_IP);
+ arp.setProtocolAddressLength((byte) Ip4Address.BYTE_LENGTH);
+
+ arp.setSenderHardwareAddress(mac.toBytes());
+ arp.setTargetHardwareAddress(MacAddress.BROADCAST.toBytes());
+ arp.setSenderProtocolAddress(ip.getIp4Address().toOctets());
+ arp.setTargetProtocolAddress(ip.getIp4Address().toOctets());
+
+ eth.setPayload(arp);
+ return eth;
+ }
+
+ /**
+ * Returns MAC address of a host with a given target IP address by asking to
+ * host service. It does not support overlapping IP.
+ *
+ * @param targetIp target ip
+ * @return mac address, or NONE mac address if it fails to find the mac
+ */
+ private MacAddress getMacFromHostService(IpAddress targetIp) {
+ checkNotNull(targetIp);
+
+ Host host = hostService.getHostsByIp(targetIp)
+ .stream()
+ .findFirst()
+ .orElse(null);
+
+ if (host != null) {
+ log.trace("Found MAC from host service for {}", targetIp.toString());
+ return host.mac();
+ } else {
+ return MacAddress.NONE;
+ }
+ }
+}
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnNodeManager.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnNodeManager.java
new file mode 100644
index 0000000..bec23da
--- /dev/null
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnNodeManager.java
@@ -0,0 +1,972 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cordvtn.impl;
+
+import com.google.common.collect.Sets;
+import com.jcraft.jsch.Session;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.IpAddress;
+import org.onlab.util.ItemNotFoundException;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.cordvtn.api.ConnectionHandler;
+import org.onosproject.cordvtn.api.CordVtnConfig;
+import org.onosproject.cordvtn.api.CordVtnNode;
+import org.onosproject.cordvtn.api.CordVtnNodeState;
+import org.onosproject.cordvtn.api.CordVtnService;
+import org.onosproject.cordvtn.api.NetworkAddress;
+import org.onosproject.cordvtn.api.SshAccessInfo;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.Port;
+import org.onosproject.net.behaviour.BridgeConfig;
+import org.onosproject.net.behaviour.BridgeName;
+import org.onosproject.net.behaviour.ControllerInfo;
+import org.onosproject.net.behaviour.DefaultTunnelDescription;
+import org.onosproject.net.behaviour.TunnelConfig;
+import org.onosproject.net.behaviour.TunnelDescription;
+import org.onosproject.net.behaviour.TunnelName;
+import org.onosproject.net.config.NetworkConfigEvent;
+import org.onosproject.net.config.NetworkConfigListener;
+import org.onosproject.net.config.NetworkConfigRegistry;
+import org.onosproject.net.config.NetworkConfigService;
+import org.onosproject.net.device.DeviceAdminService;
+import org.onosproject.net.device.DeviceEvent;
+import org.onosproject.net.device.DeviceListener;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverHandler;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.FlowRuleService;
+import org.onosproject.net.group.GroupService;
+import org.onosproject.net.host.HostService;
+import org.onosproject.ovsdb.controller.OvsdbClientService;
+import org.onosproject.ovsdb.controller.OvsdbController;
+import org.onosproject.ovsdb.controller.OvsdbNodeId;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.MapEvent;
+import org.onosproject.store.service.MapEventListener;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.Versioned;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.net.Device.Type.SWITCH;
+import static org.onosproject.net.behaviour.TunnelDescription.Type.VXLAN;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Reads node information from the network config file and handles the config
+ * update events.
+ * Only a leader controller performs the node addition or deletion.
+ */
+@Component(immediate = true)
+@Service(value = CordVtnNodeManager.class)
+public class CordVtnNodeManager {
+
+ protected final Logger log = getLogger(getClass());
+
+ private static final KryoNamespace.Builder NODE_SERIALIZER = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(KryoNamespaces.MISC)
+ .register(CordVtnNode.class)
+ .register(NodeState.class)
+ .register(SshAccessInfo.class)
+ .register(NetworkAddress.class);
+
+ private static final String DEFAULT_BRIDGE = "br-int";
+ private static final String DEFAULT_TUNNEL = "vxlan";
+ private static final String VPORT_PREFIX = "tap";
+ private static final String OK = "OK";
+ private static final String NO = "NO";
+
+ private static final Map<String, String> DEFAULT_TUNNEL_OPTIONS = new HashMap<String, String>() {
+ {
+ put("key", "flow");
+ put("remote_ip", "flow");
+ }
+ };
+ private static final int DPID_BEGIN = 3;
+ private static final int OFPORT = 6653;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected NetworkConfigRegistry configRegistry;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected NetworkConfigService configService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected StorageService storageService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DeviceAdminService adminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OvsdbController controller;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DriverService driverService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected HostService hostService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected FlowRuleService flowRuleService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected GroupService groupService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CordVtnService cordVtnService;
+
+ private final ExecutorService eventExecutor =
+ newSingleThreadScheduledExecutor(groupedThreads("onos/cordvtncfg", "event-handler"));
+
+ private final NetworkConfigListener configListener = new InternalConfigListener();
+ private final DeviceListener deviceListener = new InternalDeviceListener();
+ private final MapEventListener<String, CordVtnNode> nodeStoreListener = new InternalMapListener();
+
+ private final OvsdbHandler ovsdbHandler = new OvsdbHandler();
+ private final BridgeHandler bridgeHandler = new BridgeHandler();
+
+ private ConsistentMap<String, CordVtnNode> nodeStore;
+ private CordVtnRuleInstaller ruleInstaller;
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ private enum NodeState implements CordVtnNodeState {
+
+ INIT {
+ @Override
+ public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
+ if (!nodeManager.isOvsdbConnected(node)) {
+ nodeManager.connectOvsdb(node);
+ } else {
+ nodeManager.createIntegrationBridge(node);
+ }
+ }
+ },
+ BRIDGE_CREATED {
+ @Override
+ public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
+ if (!nodeManager.isOvsdbConnected(node)) {
+ nodeManager.connectOvsdb(node);
+ } else {
+ nodeManager.createTunnelInterface(node);
+ nodeManager.addDataPlaneInterface(node);
+ }
+ }
+ },
+ PORTS_ADDED {
+ @Override
+ public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
+ nodeManager.setIpAddress(node);
+ }
+ },
+ COMPLETE {
+ @Override
+ public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
+ nodeManager.postInit(node);
+ }
+ },
+ INCOMPLETE {
+ @Override
+ public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
+ }
+ };
+
+ public abstract void process(CordVtnNodeManager nodeManager, CordVtnNode node);
+ }
+
+ @Activate
+ protected void active() {
+ appId = coreService.getAppId(CordVtnService.CORDVTN_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+
+ nodeStore = storageService.<String, CordVtnNode>consistentMapBuilder()
+ .withSerializer(Serializer.using(NODE_SERIALIZER.build()))
+ .withName("cordvtn-nodestore")
+ .withApplicationId(appId)
+ .build();
+
+ ruleInstaller = new CordVtnRuleInstaller(appId, flowRuleService,
+ deviceService,
+ driverService,
+ groupService,
+ configRegistry,
+ DEFAULT_TUNNEL);
+
+ nodeStore.addListener(nodeStoreListener);
+ deviceService.addListener(deviceListener);
+ configService.addListener(configListener);
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ configService.removeListener(configListener);
+ deviceService.removeListener(deviceListener);
+
+ nodeStore.removeListener(nodeStoreListener);
+ nodeStore.clear();
+
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+ }
+
+ /**
+ * Adds or updates a new node to the service.
+ *
+ * @param node cordvtn node
+ */
+ public void addOrUpdateNode(CordVtnNode node) {
+ checkNotNull(node);
+ nodeStore.put(node.hostname(), CordVtnNode.getUpdatedNode(node, getNodeState(node)));
+ }
+
+ /**
+ * Deletes a node from the service.
+ *
+ * @param node cordvtn node
+ */
+ public void deleteNode(CordVtnNode node) {
+ checkNotNull(node);
+
+ if (isOvsdbConnected(node)) {
+ disconnectOvsdb(node);
+ }
+
+ nodeStore.remove(node.hostname());
+ }
+
+ /**
+ * Initiates node to serve virtual tenant network.
+ *
+ * @param node cordvtn node
+ */
+ private void initNode(CordVtnNode node) {
+ checkNotNull(node);
+
+ NodeState state = (NodeState) node.state();
+ log.debug("Processing node: {} state: {}", node.hostname(), state);
+
+ state.process(this, node);
+ }
+
+ /**
+ * Returns node initialization state.
+ *
+ * @param node cordvtn node
+ * @return true if initial node setup is completed, otherwise false
+ */
+ public boolean isNodeInitComplete(CordVtnNode node) {
+ checkNotNull(node);
+ return nodeStore.containsKey(node.hostname()) && getNodeState(node).equals(NodeState.COMPLETE);
+ }
+
+ /**
+ * Flush flows installed by cordvtn.
+ */
+ public void flushRules() {
+ ruleInstaller.flushRules();
+ }
+
+ /**
+ * Returns if current node state saved in nodeStore is COMPLETE or not.
+ *
+ * @param node cordvtn node
+ * @return true if it's complete state, otherwise false
+ */
+ private boolean isNodeStateComplete(CordVtnNode node) {
+ checkNotNull(node);
+
+ // the state saved in nodeStore can be wrong if IP address settings are changed
+ // after the node init has been completed since there's no way to detect it
+ // getNodeState and checkNodeInitState always return correct answer but can be slow
+ Versioned<CordVtnNode> versionedNode = nodeStore.get(node.hostname());
+ CordVtnNodeState state = versionedNode.value().state();
+ return state != null && state.equals(NodeState.COMPLETE);
+ }
+
+ /**
+ * Returns detailed node initialization state.
+ *
+ * @param node cordvtn node
+ * @return string including detailed node init state
+ */
+ public String checkNodeInitState(CordVtnNode node) {
+ checkNotNull(node);
+
+ if (!nodeStore.containsKey(node.hostname())) {
+ log.warn("Node {} does not exist, add node first", node.hostname());
+ return null;
+ }
+
+ Session session = RemoteIpCommandUtil.connect(node.sshInfo());
+ if (session == null) {
+ log.debug("Failed to SSH to {}", node.hostname());
+ return null;
+ }
+
+ Set<IpAddress> intBrIps = RemoteIpCommandUtil.getCurrentIps(session, DEFAULT_BRIDGE);
+ String result = String.format(
+ "br-int created and connected : %s (%s)%n" +
+ "VXLAN interface created : %s%n" +
+ "Data plane interface added : %s (%s)%n" +
+ "IP flushed from %s : %s%n" +
+ "Data plane IP added to br-int : %s (%s)%n" +
+ "Local management IP added to br-int : %s (%s)",
+ isBrIntCreated(node) ? OK : NO, node.intBrId(),
+ isTunnelIntfCreated(node) ? OK : NO,
+ isDataPlaneIntfAdded(node) ? OK : NO, node.dpIntf(),
+ node.dpIntf(),
+ RemoteIpCommandUtil.getCurrentIps(session, node.dpIntf()).isEmpty() ? OK : NO,
+ intBrIps.contains(node.dpIp().ip()) ? OK : NO, node.dpIp().cidr(),
+ intBrIps.contains(node.localMgmtIp().ip()) ? OK : NO, node.localMgmtIp().cidr());
+
+ RemoteIpCommandUtil.disconnect(session);
+
+ return result;
+ }
+
+ /**
+ * Returns the number of the nodes known to the service.
+ *
+ * @return number of nodes
+ */
+ public int getNodeCount() {
+ return nodeStore.size();
+ }
+
+ /**
+ * Returns all nodes known to the service.
+ *
+ * @return list of nodes
+ */
+ public List<CordVtnNode> getNodes() {
+ return nodeStore.values().stream()
+ .map(Versioned::value)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Returns cordvtn node associated with a given OVSDB device.
+ *
+ * @param ovsdbId OVSDB device id
+ * @return cordvtn node, null if it fails to find the node
+ */
+ private CordVtnNode getNodeByOvsdbId(DeviceId ovsdbId) {
+ return getNodes().stream()
+ .filter(node -> node.ovsdbId().equals(ovsdbId))
+ .findFirst().orElse(null);
+ }
+
+ /**
+ * Returns cordvtn node associated with a given integration bridge.
+ *
+ * @param bridgeId device id of integration bridge
+ * @return cordvtn node, null if it fails to find the node
+ */
+ private CordVtnNode getNodeByBridgeId(DeviceId bridgeId) {
+ return getNodes().stream()
+ .filter(node -> node.intBrId().equals(bridgeId))
+ .findFirst().orElse(null);
+ }
+
+ /**
+ * Sets a new state for a given cordvtn node.
+ *
+ * @param node cordvtn node
+ * @param newState new node state
+ */
+ private void setNodeState(CordVtnNode node, NodeState newState) {
+ checkNotNull(node);
+
+ log.debug("Changed {} state: {}", node.hostname(), newState);
+ nodeStore.put(node.hostname(), CordVtnNode.getUpdatedNode(node, newState));
+ }
+
+ /**
+ * Checks current state of a given cordvtn node and returns it.
+ *
+ * @param node cordvtn node
+ * @return node state
+ */
+ private NodeState getNodeState(CordVtnNode node) {
+ checkNotNull(node);
+
+ if (isBrIntCreated(node) && isTunnelIntfCreated(node) &&
+ isDataPlaneIntfAdded(node) && isIpAddressSet(node)) {
+ return NodeState.COMPLETE;
+ } else if (isDataPlaneIntfAdded(node) && isTunnelIntfCreated(node)) {
+ return NodeState.PORTS_ADDED;
+ } else if (isBrIntCreated(node)) {
+ return NodeState.BRIDGE_CREATED;
+ } else {
+ return NodeState.INIT;
+ }
+ }
+
+ /**
+ * Performs tasks after node initialization.
+ * It disconnects unnecessary OVSDB connection and installs initial flow
+ * rules on the device.
+ *
+ * @param node cordvtn node
+ */
+ private void postInit(CordVtnNode node) {
+ disconnectOvsdb(node);
+
+ ruleInstaller.init(node.intBrId(), node.dpIntf(), node.dpIp().ip());
+
+ // add existing hosts to the service
+ deviceService.getPorts(node.intBrId()).stream()
+ .filter(port -> getPortName(port).startsWith(VPORT_PREFIX) &&
+ port.isEnabled())
+ .forEach(port -> cordVtnService.addServiceVm(node, getConnectPoint(port)));
+
+ // remove stale hosts from the service
+ hostService.getHosts().forEach(host -> {
+ Port port = deviceService.getPort(host.location().deviceId(), host.location().port());
+ if (port == null) {
+ cordVtnService.removeServiceVm(getConnectPoint(host));
+ }
+ });
+
+ log.info("Finished init {}", node.hostname());
+ }
+
+ /**
+ * Returns port name.
+ *
+ * @param port port
+ * @return port name
+ */
+ private String getPortName(Port port) {
+ return port.annotations().value("portName");
+ }
+
+ /**
+ * Returns connection state of OVSDB server for a given node.
+ *
+ * @param node cordvtn node
+ * @return true if it is connected, false otherwise
+ */
+ private boolean isOvsdbConnected(CordVtnNode node) {
+ checkNotNull(node);
+
+ OvsdbClientService ovsdbClient = getOvsdbClient(node);
+ return deviceService.isAvailable(node.ovsdbId()) &&
+ ovsdbClient != null && ovsdbClient.isConnected();
+ }
+
+ /**
+ * Connects to OVSDB server for a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void connectOvsdb(CordVtnNode node) {
+ checkNotNull(node);
+
+ if (!nodeStore.containsKey(node.hostname())) {
+ log.warn("Node {} does not exist", node.hostname());
+ return;
+ }
+
+ if (!isOvsdbConnected(node)) {
+ controller.connect(node.hostMgmtIp().ip(), node.ovsdbPort());
+ }
+ }
+
+ /**
+ * Disconnects OVSDB server for a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void disconnectOvsdb(CordVtnNode node) {
+ checkNotNull(node);
+
+ if (!nodeStore.containsKey(node.hostname())) {
+ log.warn("Node {} does not exist", node.hostname());
+ return;
+ }
+
+ if (isOvsdbConnected(node)) {
+ OvsdbClientService ovsdbClient = getOvsdbClient(node);
+ ovsdbClient.disconnect();
+ }
+ }
+
+ /**
+ * Returns OVSDB client for a given node.
+ *
+ * @param node cordvtn node
+ * @return OVSDB client, or null if it fails to get OVSDB client
+ */
+ private OvsdbClientService getOvsdbClient(CordVtnNode node) {
+ checkNotNull(node);
+
+ OvsdbClientService ovsdbClient = controller.getOvsdbClient(
+ new OvsdbNodeId(node.hostMgmtIp().ip(), node.ovsdbPort().toInt()));
+ if (ovsdbClient == null) {
+ log.trace("Couldn't find OVSDB client for {}", node.hostname());
+ }
+ return ovsdbClient;
+ }
+
+ /**
+ * Creates an integration bridge for a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void createIntegrationBridge(CordVtnNode node) {
+ if (isBrIntCreated(node)) {
+ return;
+ }
+
+ List<ControllerInfo> controllers = new ArrayList<>();
+ Sets.newHashSet(clusterService.getNodes()).stream()
+ .forEach(controller -> {
+ ControllerInfo ctrlInfo = new ControllerInfo(controller.ip(), OFPORT, "tcp");
+ controllers.add(ctrlInfo);
+ });
+
+ String dpid = node.intBrId().toString().substring(DPID_BEGIN);
+
+ try {
+ DriverHandler handler = driverService.createHandler(node.ovsdbId());
+ BridgeConfig bridgeConfig = handler.behaviour(BridgeConfig.class);
+ bridgeConfig.addBridge(BridgeName.bridgeName(DEFAULT_BRIDGE), dpid, controllers);
+ } catch (ItemNotFoundException e) {
+ log.warn("Failed to create integration bridge on {}", node.hostname());
+ }
+ }
+
+ /**
+ * Creates tunnel interface to the integration bridge for a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void createTunnelInterface(CordVtnNode node) {
+ if (isTunnelIntfCreated(node)) {
+ return;
+ }
+
+ DefaultAnnotations.Builder optionBuilder = DefaultAnnotations.builder();
+ for (String key : DEFAULT_TUNNEL_OPTIONS.keySet()) {
+ optionBuilder.set(key, DEFAULT_TUNNEL_OPTIONS.get(key));
+ }
+
+ TunnelDescription description = new DefaultTunnelDescription(
+ null, null, VXLAN, TunnelName.tunnelName(DEFAULT_TUNNEL),
+ optionBuilder.build());
+
+ try {
+ DriverHandler handler = driverService.createHandler(node.ovsdbId());
+ TunnelConfig tunnelConfig = handler.behaviour(TunnelConfig.class);
+ tunnelConfig.createTunnelInterface(BridgeName.bridgeName(DEFAULT_BRIDGE), description);
+ } catch (ItemNotFoundException e) {
+ log.warn("Failed to create tunnel interface on {}", node.hostname());
+ }
+ }
+
+ /**
+ * Adds data plane interface to a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void addDataPlaneInterface(CordVtnNode node) {
+ if (isDataPlaneIntfAdded(node)) {
+ return;
+ }
+
+ try {
+ DriverHandler handler = driverService.createHandler(node.ovsdbId());
+ BridgeConfig bridgeConfig = handler.behaviour(BridgeConfig.class);
+ bridgeConfig.addPort(BridgeName.bridgeName(DEFAULT_BRIDGE), node.dpIntf());
+ } catch (ItemNotFoundException e) {
+ log.warn("Failed to add {} on {}", node.dpIntf(), node.hostname());
+ }
+ }
+
+ /**
+ * Flushes IP address from data plane interface and adds data plane IP address
+ * to integration bridge.
+ *
+ * @param node cordvtn node
+ */
+ private void setIpAddress(CordVtnNode node) {
+ Session session = RemoteIpCommandUtil.connect(node.sshInfo());
+ if (session == null) {
+ log.debug("Failed to SSH to {}", node.hostname());
+ return;
+ }
+
+ RemoteIpCommandUtil.getCurrentIps(session, DEFAULT_BRIDGE).stream()
+ .filter(ip -> !ip.equals(node.localMgmtIp().ip()))
+ .filter(ip -> !ip.equals(node.dpIp().ip()))
+ .forEach(ip -> RemoteIpCommandUtil.deleteIp(session, ip, DEFAULT_BRIDGE));
+
+ boolean result = RemoteIpCommandUtil.flushIp(session, node.dpIntf()) &&
+ RemoteIpCommandUtil.setInterfaceUp(session, node.dpIntf()) &&
+ RemoteIpCommandUtil.addIp(session, node.dpIp(), DEFAULT_BRIDGE) &&
+ RemoteIpCommandUtil.addIp(session, node.localMgmtIp(), DEFAULT_BRIDGE) &&
+ RemoteIpCommandUtil.setInterfaceUp(session, DEFAULT_BRIDGE);
+
+ RemoteIpCommandUtil.disconnect(session);
+
+ if (result) {
+ setNodeState(node, NodeState.COMPLETE);
+ }
+ }
+
+ /**
+ * Checks if integration bridge exists and available.
+ *
+ * @param node cordvtn node
+ * @return true if the bridge is available, false otherwise
+ */
+ private boolean isBrIntCreated(CordVtnNode node) {
+ return (deviceService.getDevice(node.intBrId()) != null
+ && deviceService.isAvailable(node.intBrId()));
+ }
+
+ /**
+ * Checks if tunnel interface exists.
+ *
+ * @param node cordvtn node
+ * @return true if the interface exists, false otherwise
+ */
+ private boolean isTunnelIntfCreated(CordVtnNode node) {
+ return deviceService.getPorts(node.intBrId())
+ .stream()
+ .filter(p -> getPortName(p).contains(DEFAULT_TUNNEL) &&
+ p.isEnabled())
+ .findAny().isPresent();
+ }
+
+ /**
+ * Checks if data plane interface exists.
+ *
+ * @param node cordvtn node
+ * @return true if the interface exists, false otherwise
+ */
+ private boolean isDataPlaneIntfAdded(CordVtnNode node) {
+ return deviceService.getPorts(node.intBrId())
+ .stream()
+ .filter(p -> getPortName(p).contains(node.dpIntf()) &&
+ p.isEnabled())
+ .findAny().isPresent();
+ }
+
+ /**
+ * Checks if the IP addresses are correctly set.
+ *
+ * @param node cordvtn node
+ * @return true if the IP is set, false otherwise
+ */
+ private boolean isIpAddressSet(CordVtnNode node) {
+ Session session = RemoteIpCommandUtil.connect(node.sshInfo());
+ if (session == null) {
+ log.debug("Failed to SSH to {}", node.hostname());
+ return false;
+ }
+
+ Set<IpAddress> intBrIps = RemoteIpCommandUtil.getCurrentIps(session, DEFAULT_BRIDGE);
+ boolean result = RemoteIpCommandUtil.getCurrentIps(session, node.dpIntf()).isEmpty() &&
+ RemoteIpCommandUtil.isInterfaceUp(session, node.dpIntf()) &&
+ intBrIps.contains(node.dpIp().ip()) &&
+ intBrIps.contains(node.localMgmtIp().ip()) &&
+ RemoteIpCommandUtil.isInterfaceUp(session, DEFAULT_BRIDGE);
+
+ RemoteIpCommandUtil.disconnect(session);
+ return result;
+ }
+
+ /**
+ * Returns connect point of a given port.
+ *
+ * @param port port
+ * @return connect point
+ */
+ private ConnectPoint getConnectPoint(Port port) {
+ return new ConnectPoint(port.element().id(), port.number());
+ }
+
+ /**
+ * Returns connect point of a given host.
+ *
+ * @param host host
+ * @return connect point
+ */
+ private ConnectPoint getConnectPoint(Host host) {
+ return new ConnectPoint(host.location().deviceId(), host.location().port());
+ }
+
+ private class OvsdbHandler implements ConnectionHandler<Device> {
+
+ @Override
+ public void connected(Device device) {
+ CordVtnNode node = getNodeByOvsdbId(device.id());
+ if (node != null) {
+ setNodeState(node, getNodeState(node));
+ } else {
+ log.debug("{} is detected on unregistered node, ignore it.", device.id());
+ }
+ }
+
+ @Override
+ public void disconnected(Device device) {
+ if (!deviceService.isAvailable(device.id())) {
+ log.debug("Device {} is disconnected", device.id());
+ adminService.removeDevice(device.id());
+ }
+ }
+ }
+
+ private class BridgeHandler implements ConnectionHandler<Device> {
+
+ @Override
+ public void connected(Device device) {
+ CordVtnNode node = getNodeByBridgeId(device.id());
+ if (node != null) {
+ setNodeState(node, getNodeState(node));
+ } else {
+ log.debug("{} is detected on unregistered node, ignore it.", device.id());
+ }
+ }
+
+ @Override
+ public void disconnected(Device device) {
+ CordVtnNode node = getNodeByBridgeId(device.id());
+ if (node != null) {
+ log.debug("Integration Bridge is disconnected from {}", node.hostname());
+ setNodeState(node, NodeState.INCOMPLETE);
+ }
+ }
+
+ /**
+ * Handles port added situation.
+ * If the added port is tunnel or data plane interface, proceed to the remaining
+ * node initialization. Otherwise, do nothing.
+ *
+ * @param port port
+ */
+ public void portAdded(Port port) {
+ CordVtnNode node = getNodeByBridgeId((DeviceId) port.element().id());
+ String portName = getPortName(port);
+
+ if (node == null) {
+ log.debug("{} is added to unregistered node, ignore it.", portName);
+ return;
+ }
+
+ log.info("Port {} is added to {}", portName, node.hostname());
+
+ if (portName.startsWith(VPORT_PREFIX)) {
+ if (isNodeStateComplete(node)) {
+ cordVtnService.addServiceVm(node, getConnectPoint(port));
+ } else {
+ log.debug("VM is detected on incomplete node, ignore it.", portName);
+ }
+ } else if (portName.contains(DEFAULT_TUNNEL) || portName.equals(node.dpIntf())) {
+ setNodeState(node, getNodeState(node));
+ }
+ }
+
+ /**
+ * Handles port removed situation.
+ * If the removed port is tunnel or data plane interface, proceed to the remaining
+ * node initialization.Others, do nothing.
+ *
+ * @param port port
+ */
+ public void portRemoved(Port port) {
+ CordVtnNode node = getNodeByBridgeId((DeviceId) port.element().id());
+ String portName = getPortName(port);
+
+ if (node == null) {
+ return;
+ }
+
+ log.info("Port {} is removed from {}", portName, node.hostname());
+
+ if (portName.startsWith(VPORT_PREFIX)) {
+ if (isNodeStateComplete(node)) {
+ cordVtnService.removeServiceVm(getConnectPoint(port));
+ } else {
+ log.debug("VM is vanished from incomplete node, ignore it.", portName);
+ }
+ } else if (portName.contains(DEFAULT_TUNNEL) || portName.equals(node.dpIntf())) {
+ setNodeState(node, NodeState.INCOMPLETE);
+ }
+ }
+ }
+
+ private class InternalDeviceListener implements DeviceListener {
+
+ @Override
+ public void event(DeviceEvent event) {
+
+ NodeId leaderNodeId = leadershipService.getLeader(appId.name());
+ if (!Objects.equals(localNodeId, leaderNodeId)) {
+ // do not allow to proceed without leadership
+ return;
+ }
+
+ Device device = event.subject();
+ ConnectionHandler<Device> handler =
+ (device.type().equals(SWITCH) ? bridgeHandler : ovsdbHandler);
+
+ switch (event.type()) {
+ case PORT_ADDED:
+ eventExecutor.execute(() -> bridgeHandler.portAdded(event.port()));
+ break;
+ case PORT_UPDATED:
+ if (!event.port().isEnabled()) {
+ eventExecutor.execute(() -> bridgeHandler.portRemoved(event.port()));
+ }
+ break;
+ case DEVICE_ADDED:
+ case DEVICE_AVAILABILITY_CHANGED:
+ if (deviceService.isAvailable(device.id())) {
+ eventExecutor.execute(() -> handler.connected(device));
+ } else {
+ eventExecutor.execute(() -> handler.disconnected(device));
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ /**
+ * Reads cordvtn nodes from config file.
+ */
+ private void readConfiguration() {
+ CordVtnConfig config = configRegistry.getConfig(appId, CordVtnConfig.class);
+ if (config == null) {
+ log.debug("No configuration found");
+ return;
+ }
+
+ config.cordVtnNodes().forEach(this::addOrUpdateNode);
+ }
+
+ private class InternalConfigListener implements NetworkConfigListener {
+
+ @Override
+ public void event(NetworkConfigEvent event) {
+ NodeId leaderNodeId = leadershipService.getLeader(appId.name());
+ if (!Objects.equals(localNodeId, leaderNodeId)) {
+ // do not allow to proceed without leadership
+ return;
+ }
+
+ if (!event.configClass().equals(CordVtnConfig.class)) {
+ return;
+ }
+
+ switch (event.type()) {
+ case CONFIG_ADDED:
+ case CONFIG_UPDATED:
+ eventExecutor.execute(CordVtnNodeManager.this::readConfiguration);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ private class InternalMapListener implements MapEventListener<String, CordVtnNode> {
+
+ @Override
+ public void event(MapEvent<String, CordVtnNode> event) {
+ NodeId leaderNodeId = leadershipService.getLeader(appId.name());
+ if (!Objects.equals(localNodeId, leaderNodeId)) {
+ // do not allow to proceed without leadership
+ return;
+ }
+
+ CordVtnNode oldNode;
+ CordVtnNode newNode;
+
+ switch (event.type()) {
+ case UPDATE:
+ oldNode = event.oldValue().value();
+ newNode = event.newValue().value();
+
+ if (!newNode.equals(oldNode)) {
+ log.info("{} has been updated", newNode.hostname());
+ log.debug("New node: {}", newNode);
+ }
+ // perform init procedure based on current state on any updates,
+ // insert, or even if the node is the same for robustness since
+ // it's no harm to run the init procedure multiple times
+ eventExecutor.execute(() -> initNode(newNode));
+ break;
+ case INSERT:
+ newNode = event.newValue().value();
+ log.info("Added {}", newNode.hostname());
+ eventExecutor.execute(() -> initNode(newNode));
+ break;
+ case REMOVE:
+ oldNode = event.oldValue().value();
+ log.info("{} is removed", oldNode.hostname());
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnRuleInstaller.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnRuleInstaller.java
new file mode 100644
index 0000000..8652e18
--- /dev/null
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/CordVtnRuleInstaller.java
@@ -0,0 +1,1539 @@
+/*
+ * Copyright 2015-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cordvtn.impl;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.Ip4Prefix;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.TpPort;
+import org.onlab.packet.VlanId;
+import org.onlab.util.ItemNotFoundException;
+import org.onosproject.cordvtn.api.CordService;
+import org.onosproject.cordvtn.api.CordServiceId;
+import org.onosproject.cordvtn.api.CordVtnConfig;
+import org.onosproject.cordvtn.api.CordVtnNode;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.DefaultGroupId;
+import org.onosproject.core.GroupId;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
+import org.onosproject.net.config.NetworkConfigRegistry;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DefaultDriverData;
+import org.onosproject.net.driver.DefaultDriverHandler;
+import org.onosproject.net.driver.Driver;
+import org.onosproject.net.driver.DriverHandler;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultFlowRule;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.FlowRule;
+import org.onosproject.net.flow.FlowRuleOperations;
+import org.onosproject.net.flow.FlowRuleOperationsContext;
+import org.onosproject.net.flow.FlowRuleService;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.EthCriterion;
+import org.onosproject.net.flow.criteria.IPCriterion;
+import org.onosproject.net.flow.criteria.PortCriterion;
+import org.onosproject.net.flow.instructions.ExtensionPropertyException;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.onosproject.net.flow.instructions.Instruction;
+import org.onosproject.net.flow.instructions.Instructions;
+import org.onosproject.net.flow.instructions.L2ModificationInstruction;
+import org.onosproject.net.flow.instructions.L2ModificationInstruction.ModEtherInstruction;
+import org.onosproject.net.group.DefaultGroupBucket;
+import org.onosproject.net.group.DefaultGroupDescription;
+import org.onosproject.net.group.DefaultGroupKey;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupBucket;
+import org.onosproject.net.group.GroupBuckets;
+import org.onosproject.net.group.GroupDescription;
+import org.onosproject.net.group.GroupKey;
+import org.onosproject.net.group.GroupService;
+import org.onosproject.openstackinterface.OpenstackNetwork;
+import org.onosproject.openstackinterface.OpenstackSubnet;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onosproject.net.flow.criteria.Criterion.Type.IN_PORT;
+import static org.onosproject.net.flow.criteria.Criterion.Type.IPV4_DST;
+import static org.onosproject.net.flow.criteria.Criterion.Type.IPV4_SRC;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST;
+import static org.onosproject.net.flow.instructions.L2ModificationInstruction.L2SubType.ETH_DST;
+import static org.onosproject.net.flow.instructions.L2ModificationInstruction.L2SubType.VLAN_PUSH;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Populates rules for CORD VTN service.
+ */
+public class CordVtnRuleInstaller {
+
+ protected final Logger log = getLogger(getClass());
+
+ private static final int TABLE_FIRST = 0;
+ private static final int TABLE_IN_PORT = 1;
+ private static final int TABLE_ACCESS_TYPE = 2;
+ private static final int TABLE_IN_SERVICE = 3;
+ private static final int TABLE_DST_IP = 4;
+ private static final int TABLE_TUNNEL_IN = 5;
+ private static final int TABLE_Q_IN_Q = 6;
+
+ private static final int MANAGEMENT_PRIORITY = 55000;
+ private static final int VSG_PRIORITY = 55000;
+ private static final int HIGH_PRIORITY = 50000;
+ private static final int DEFAULT_PRIORITY = 5000;
+ private static final int LOW_PRIORITY = 4000;
+ private static final int LOWEST_PRIORITY = 0;
+
+ private static final int VXLAN_UDP_PORT = 4789;
+ private static final VlanId VLAN_WAN = VlanId.vlanId((short) 500);
+
+ private static final String PORT_NAME = "portName";
+ private static final String DATA_PLANE_INTF = "dataPlaneIntf";
+ private static final String S_TAG = "stag";
+
+ private final ApplicationId appId;
+ private final FlowRuleService flowRuleService;
+ private final DeviceService deviceService;
+ private final DriverService driverService;
+ private final GroupService groupService;
+ private final NetworkConfigRegistry configRegistry;
+ private final String tunnelType;
+
+ /**
+ * Creates a new rule populator.
+ *
+ * @param appId application id
+ * @param flowRuleService flow rule service
+ * @param deviceService device service
+ * @param driverService driver service
+ * @param groupService group service
+ * @param configRegistry config registry
+ * @param tunnelType tunnel type
+ */
+ public CordVtnRuleInstaller(ApplicationId appId,
+ FlowRuleService flowRuleService,
+ DeviceService deviceService,
+ DriverService driverService,
+ GroupService groupService,
+ NetworkConfigRegistry configRegistry,
+ String tunnelType) {
+ this.appId = appId;
+ this.flowRuleService = flowRuleService;
+ this.deviceService = deviceService;
+ this.driverService = driverService;
+ this.groupService = groupService;
+ this.configRegistry = configRegistry;
+ this.tunnelType = checkNotNull(tunnelType);
+ }
+
+ /**
+ * Installs table miss rule to a give device.
+ *
+ * @param deviceId device id to install the rules
+ * @param dpIntf data plane interface name
+ * @param dpIp data plane ip address
+ */
+ public void init(DeviceId deviceId, String dpIntf, IpAddress dpIp) {
+ // default is drop packets which can be accomplished without
+ // a table miss entry for all table.
+ PortNumber tunnelPort = getTunnelPort(deviceId);
+ PortNumber dpPort = getDpPort(deviceId, dpIntf);
+
+ processFirstTable(deviceId, dpPort, dpIp);
+ processInPortTable(deviceId, tunnelPort, dpPort);
+ processAccessTypeTable(deviceId, dpPort);
+ processQInQTable(deviceId, dpPort);
+ }
+
+ /**
+ * Flush flows installed by this application.
+ */
+ public void flushRules() {
+ flowRuleService.getFlowRulesById(appId).forEach(flowRule -> processFlowRule(false, flowRule));
+ }
+
+ /**
+ * Populates basic rules that connect a VM to the other VMs in the system.
+ *
+ * @param host host
+ * @param tunnelIp tunnel ip
+ * @param vNet openstack network
+ */
+ public void populateBasicConnectionRules(Host host, IpAddress tunnelIp, OpenstackNetwork vNet) {
+ checkNotNull(host);
+ checkNotNull(vNet);
+
+ DeviceId deviceId = host.location().deviceId();
+ PortNumber inPort = host.location().port();
+ MacAddress dstMac = host.mac();
+ IpAddress hostIp = host.ipAddresses().stream().findFirst().get();
+ long tunnelId = Long.parseLong(vNet.segmentId());
+
+ OpenstackSubnet subnet = vNet.subnets().stream()
+ .findFirst()
+ .orElse(null);
+
+ if (subnet == null) {
+ log.error("Failed to get subnet for {}", host.id());
+ return;
+ }
+
+ populateLocalInPortRule(deviceId, inPort, hostIp);
+ populateDirectAccessRule(Ip4Prefix.valueOf(subnet.cidr()), Ip4Prefix.valueOf(subnet.cidr()));
+ populateServiceIsolationRule(Ip4Prefix.valueOf(subnet.cidr()));
+ populateDstIpRule(deviceId, inPort, dstMac, hostIp, tunnelId, tunnelIp);
+ populateTunnelInRule(deviceId, inPort, dstMac, tunnelId);
+ }
+
+ /**
+ * Removes all rules related to a given service VM host.
+ *
+ * @param host host to be removed
+ */
+ public void removeBasicConnectionRules(Host host) {
+ checkNotNull(host);
+
+ DeviceId deviceId = host.location().deviceId();
+ MacAddress mac = host.mac();
+ PortNumber port = host.location().port();
+ IpAddress ip = host.ipAddresses().stream().findFirst().orElse(null);
+
+ for (FlowRule flowRule : flowRuleService.getFlowRulesById(appId)) {
+ if (flowRule.deviceId().equals(deviceId)) {
+ PortNumber inPort = getInPort(flowRule);
+ if (inPort != null && inPort.equals(port)) {
+ processFlowRule(false, flowRule);
+ continue;
+ }
+
+ PortNumber output = getOutputFromTreatment(flowRule);
+ if (output != null && output.equals(host.location().port())) {
+ processFlowRule(false, flowRule);
+ }
+ }
+
+ MacAddress dstMac = getDstMacFromTreatment(flowRule);
+ if (dstMac != null && dstMac.equals(mac)) {
+ processFlowRule(false, flowRule);
+ continue;
+ }
+
+ dstMac = getDstMacFromSelector(flowRule);
+ if (dstMac != null && dstMac.equals(mac)) {
+ processFlowRule(false, flowRule);
+ continue;
+ }
+
+ IpPrefix dstIp = getDstIpFromSelector(flowRule);
+ if (dstIp != null && dstIp.equals(ip.toIpPrefix())) {
+ processFlowRule(false, flowRule);
+ }
+ }
+
+ // TODO uninstall same network access rule in access table if no vm exists in the network
+ }
+
+ /**
+ * Populates service dependency rules.
+ *
+ * @param tService tenant cord service
+ * @param pService provider cord service
+ * @param isBidirectional true to enable bidirectional connection between two services
+ */
+ public void populateServiceDependencyRules(CordService tService, CordService pService,
+ boolean isBidirectional) {
+ checkNotNull(tService);
+ checkNotNull(pService);
+
+ Ip4Prefix srcRange = tService.serviceIpRange().getIp4Prefix();
+ Ip4Prefix dstRange = pService.serviceIpRange().getIp4Prefix();
+ Ip4Address serviceIp = pService.serviceIp().getIp4Address();
+
+ Map<DeviceId, GroupId> outGroups = Maps.newHashMap();
+ Map<DeviceId, Set<PortNumber>> inPorts = Maps.newHashMap();
+
+ getVirtualSwitches().stream().forEach(deviceId -> {
+ GroupId groupId = createServiceGroup(deviceId, pService);
+ outGroups.put(deviceId, groupId);
+
+ Set<PortNumber> vms = tService.hosts().keySet()
+ .stream()
+ .filter(host -> host.location().deviceId().equals(deviceId))
+ .map(host -> host.location().port())
+ .collect(Collectors.toSet());
+ inPorts.put(deviceId, vms);
+ });
+
+ populateIndirectAccessRule(srcRange, serviceIp, outGroups);
+ populateDirectAccessRule(srcRange, dstRange);
+ if (isBidirectional) {
+ populateDirectAccessRule(dstRange, srcRange);
+ }
+ populateInServiceRule(inPorts, outGroups);
+ }
+
+ /**
+ * Removes service dependency rules.
+ *
+ * @param tService tenant cord service
+ * @param pService provider cord service
+ */
+ public void removeServiceDependencyRules(CordService tService, CordService pService) {
+ checkNotNull(tService);
+ checkNotNull(pService);
+
+ Ip4Prefix srcRange = tService.serviceIpRange().getIp4Prefix();
+ Ip4Prefix dstRange = pService.serviceIpRange().getIp4Prefix();
+ IpPrefix serviceIp = pService.serviceIp().toIpPrefix();
+
+ Map<DeviceId, GroupId> outGroups = Maps.newHashMap();
+ GroupKey groupKey = new DefaultGroupKey(pService.id().id().getBytes());
+
+ getVirtualSwitches().stream().forEach(deviceId -> {
+ Group group = groupService.getGroup(deviceId, groupKey);
+ if (group != null) {
+ outGroups.put(deviceId, group.id());
+ }
+ });
+
+ for (FlowRule flowRule : flowRuleService.getFlowRulesById(appId)) {
+ IpPrefix dstIp = getDstIpFromSelector(flowRule);
+ IpPrefix srcIp = getSrcIpFromSelector(flowRule);
+
+ if (dstIp != null && dstIp.equals(serviceIp)) {
+ processFlowRule(false, flowRule);
+ continue;
+ }
+
+ if (dstIp != null && srcIp != null) {
+ if (dstIp.equals(dstRange) && srcIp.equals(srcRange)) {
+ processFlowRule(false, flowRule);
+ continue;
+ }
+
+ if (dstIp.equals(srcRange) && srcIp.equals(dstRange)) {
+ processFlowRule(false, flowRule);
+ continue;
+ }
+ }
+
+ GroupId groupId = getGroupIdFromTreatment(flowRule);
+ if (groupId != null && groupId.equals(outGroups.get(flowRule.deviceId()))) {
+ processFlowRule(false, flowRule);
+ }
+ }
+
+ // TODO remove the group if it is not in use
+ }
+
+ /**
+ * Updates group buckets for a given service to all devices.
+ *
+ * @param service cord service
+ */
+ public void updateServiceGroup(CordService service) {
+ checkNotNull(service);
+
+ GroupKey groupKey = getGroupKey(service.id());
+
+ for (DeviceId deviceId : getVirtualSwitches()) {
+ Group group = groupService.getGroup(deviceId, groupKey);
+ if (group == null) {
+ log.trace("No group exists for service {} in {}, do nothing.", service.id(), deviceId);
+ continue;
+ }
+
+ List<GroupBucket> oldBuckets = group.buckets().buckets();
+ List<GroupBucket> newBuckets = getServiceGroupBuckets(
+ deviceId, service.segmentationId(), service.hosts()).buckets();
+
+ if (oldBuckets.equals(newBuckets)) {
+ continue;
+ }
+
+ List<GroupBucket> bucketsToRemove = new ArrayList<>(oldBuckets);
+ bucketsToRemove.removeAll(newBuckets);
+ if (!bucketsToRemove.isEmpty()) {
+ groupService.removeBucketsFromGroup(
+ deviceId,
+ groupKey,
+ new GroupBuckets(bucketsToRemove),
+ groupKey, appId);
+ }
+
+ List<GroupBucket> bucketsToAdd = new ArrayList<>(newBuckets);
+ bucketsToAdd.removeAll(oldBuckets);
+ if (!bucketsToAdd.isEmpty()) {
+ groupService.addBucketsToGroup(
+ deviceId,
+ groupKey,
+ new GroupBuckets(bucketsToAdd),
+ groupKey, appId);
+ }
+ }
+ }
+
+ /**
+ * Populates flow rules for management network access.
+ *
+ * @param host host which has management network interface
+ * @param mService management network service
+ */
+ public void populateManagementNetworkRules(Host host, CordService mService) {
+ checkNotNull(mService);
+
+ DeviceId deviceId = host.location().deviceId();
+ IpAddress hostIp = host.ipAddresses().stream().findFirst().get();
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_ARP)
+ .matchArpTpa(mService.serviceIp().getIp4Address())
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(PortNumber.LOCAL)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(MANAGEMENT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ selector = DefaultTrafficSelector.builder()
+ .matchInPort(PortNumber.LOCAL)
+ .matchEthType(Ethernet.TYPE_ARP)
+ .matchArpTpa(hostIp.getIp4Address())
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .setOutput(host.location().port())
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(MANAGEMENT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ selector = DefaultTrafficSelector.builder()
+ .matchInPort(PortNumber.LOCAL)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(mService.serviceIpRange())
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .transition(TABLE_DST_IP)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(MANAGEMENT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(mService.serviceIp().toIpPrefix())
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .setOutput(PortNumber.LOCAL)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(MANAGEMENT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_ACCESS_TYPE)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+
+ /**
+ * Removes management network access rules.
+ *
+ * @param host host to be removed
+ * @param mService service for management network
+ */
+ public void removeManagementNetworkRules(Host host, CordService mService) {
+ checkNotNull(mService);
+ // TODO remove management network specific rules
+ }
+
+ /**
+ * Populates rules for vSG VM.
+ *
+ * @param vSgHost vSG host
+ * @param vSgIps set of ip addresses of vSGs running inside the vSG VM
+ */
+ public void populateSubscriberGatewayRules(Host vSgHost, Set<IpAddress> vSgIps) {
+ VlanId serviceVlan = getServiceVlan(vSgHost);
+ PortNumber dpPort = getDpPort(vSgHost);
+
+ if (serviceVlan == null || dpPort == null) {
+ log.warn("Failed to populate rules for vSG VM {}", vSgHost.id());
+ return;
+ }
+
+ // for traffics with s-tag, strip the tag and take through the vSG VM
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(dpPort)
+ .matchVlanId(serviceVlan)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(vSgHost.location().port())
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(vSgHost.location().deviceId())
+ .forTable(TABLE_Q_IN_Q)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ // for traffics with customer vlan, tag with the service vlan based on input port with
+ // lower priority to avoid conflict with WAN tag
+ selector = DefaultTrafficSelector.builder()
+ .matchInPort(vSgHost.location().port())
+ .matchVlanId(serviceVlan)
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .setOutput(dpPort)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(vSgHost.location().deviceId())
+ .forTable(TABLE_Q_IN_Q)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ // for traffic coming from WAN, tag 500 and take through the vSG VM
+ // based on destination ip
+ vSgIps.stream().forEach(ip -> {
+ TrafficSelector downstream = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(ip.toIpPrefix())
+ .build();
+
+ TrafficTreatment downstreamTreatment = DefaultTrafficTreatment.builder()
+ .pushVlan()
+ .setVlanId(VLAN_WAN)
+ .setEthDst(vSgHost.mac())
+ .setOutput(vSgHost.location().port())
+ .build();
+
+ FlowRule downstreamFlowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(downstream)
+ .withTreatment(downstreamTreatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(vSgHost.location().deviceId())
+ .forTable(TABLE_DST_IP)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, downstreamFlowRule);
+ });
+
+ // remove downstream flow rules for the vSG not shown in vSgIps
+ for (FlowRule rule : flowRuleService.getFlowRulesById(appId)) {
+ if (!rule.deviceId().equals(vSgHost.location().deviceId())) {
+ continue;
+ }
+ PortNumber output = getOutputFromTreatment(rule);
+ if (output == null || !output.equals(vSgHost.location().port()) ||
+ !isVlanPushFromTreatment(rule)) {
+ continue;
+ }
+
+ IpPrefix dstIp = getDstIpFromSelector(rule);
+ if (dstIp != null && !vSgIps.contains(dstIp.address())) {
+ processFlowRule(false, rule);
+ }
+ }
+ }
+
+ /**
+ * Populates default rules on the first table.
+ * It includes the rules for shuttling vxlan-encapped packets between ovs and
+ * linux stack,and external network connectivity.
+ *
+ * @param deviceId device id
+ * @param dpPort data plane interface port number
+ * @param dpIp data plane ip address
+ */
+ private void processFirstTable(DeviceId deviceId, PortNumber dpPort, IpAddress dpIp) {
+ // take vxlan packet out onto the physical port
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(PortNumber.LOCAL)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(dpPort)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(HIGH_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ // take a vxlan encap'd packet through the Linux stack
+ selector = DefaultTrafficSelector.builder()
+ .matchInPort(dpPort)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(IPv4.PROTOCOL_UDP)
+ .matchUdpDst(TpPort.tpPort(VXLAN_UDP_PORT))
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .setOutput(PortNumber.LOCAL)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(HIGH_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ // take a packet to the data plane ip through Linux stack
+ selector = DefaultTrafficSelector.builder()
+ .matchInPort(dpPort)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(dpIp.toIpPrefix())
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .setOutput(PortNumber.LOCAL)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(HIGH_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ // take an arp packet from physical through Linux stack
+ selector = DefaultTrafficSelector.builder()
+ .matchInPort(dpPort)
+ .matchEthType(Ethernet.TYPE_ARP)
+ .matchArpTpa(dpIp.getIp4Address())
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .setOutput(PortNumber.LOCAL)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(HIGH_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ // take all else to the next table
+ selector = DefaultTrafficSelector.builder()
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .transition(TABLE_IN_PORT)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(LOWEST_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ // take all vlan tagged packet to the Q_IN_Q table
+ selector = DefaultTrafficSelector.builder()
+ .matchVlanId(VlanId.ANY)
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .transition(TABLE_Q_IN_Q)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(VSG_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_FIRST)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+
+ /**
+ * Forward table miss packets in ACCESS_TYPE table to data plane port.
+ *
+ * @param deviceId device id
+ * @param dpPort data plane interface port number
+ */
+ private void processAccessTypeTable(DeviceId deviceId, PortNumber dpPort) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(dpPort)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(LOWEST_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_ACCESS_TYPE)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+
+ /**
+ * Populates default rules for IN_PORT table.
+ * All packets from tunnel port are forwarded to TUNNEL_ID table and all packets
+ * from data plane interface port to ACCESS_TYPE table.
+ *
+ * @param deviceId device id to install the rules
+ * @param tunnelPort tunnel port number
+ * @param dpPort data plane interface port number
+ */
+ private void processInPortTable(DeviceId deviceId, PortNumber tunnelPort, PortNumber dpPort) {
+ checkNotNull(tunnelPort);
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(tunnelPort)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .transition(TABLE_TUNNEL_IN)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_IN_PORT)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ selector = DefaultTrafficSelector.builder()
+ .matchInPort(dpPort)
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .transition(TABLE_DST_IP)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_IN_PORT)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+
+ /**
+ * Populates default rules for Q_IN_Q table.
+ *
+ * @param deviceId device id
+ * @param dpPort data plane interface port number
+ */
+ private void processQInQTable(DeviceId deviceId, PortNumber dpPort) {
+ // for traffic going out to WAN, strip vid 500 and take through data plane interface
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchVlanId(VLAN_WAN)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .popVlan()
+ .setOutput(dpPort)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_Q_IN_Q)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ selector = DefaultTrafficSelector.builder()
+ .matchVlanId(VLAN_WAN)
+ .matchEthType(Ethernet.TYPE_ARP)
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .setOutput(PortNumber.CONTROLLER)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(HIGH_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_Q_IN_Q)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+
+ /**
+ * Populates rules for local in port in IN_PORT table.
+ * Flows from a given in port, whose source IP is service IP transition
+ * to DST_TYPE table. Other flows transition to IN_SERVICE table.
+ *
+ * @param deviceId device id to install the rules
+ * @param inPort in port
+ * @param srcIp source ip
+ */
+ private void populateLocalInPortRule(DeviceId deviceId, PortNumber inPort, IpAddress srcIp) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(inPort)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(srcIp.toIpPrefix())
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .transition(TABLE_ACCESS_TYPE)
+ .build();
+
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_IN_PORT)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ selector = DefaultTrafficSelector.builder()
+ .matchInPort(inPort)
+ .build();
+
+ treatment = DefaultTrafficTreatment.builder()
+ .transition(TABLE_IN_SERVICE)
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(LOW_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_IN_PORT)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+
+ /**
+ * Populates direct VM access rules for ACCESS_TYPE table.
+ * These rules are installed to all devices.
+ *
+ * @param srcRange source ip range
+ * @param dstRange destination ip range
+ */
+ private void populateDirectAccessRule(Ip4Prefix srcRange, Ip4Prefix dstRange) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(srcRange)
+ .matchIPDst(dstRange)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .transition(TABLE_DST_IP)
+ .build();
+
+
+ getVirtualSwitches().stream().forEach(deviceId -> {
+ FlowRule flowRuleDirect = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_ACCESS_TYPE)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRuleDirect);
+ });
+ }
+
+ /**
+ * Populates drop rules that does not match any direct access rules but has
+ * destination to a different service network in ACCESS_TYPE table.
+ *
+ * @param dstRange destination ip range
+ */
+ private void populateServiceIsolationRule(Ip4Prefix dstRange) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(dstRange)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .drop()
+ .build();
+
+ getVirtualSwitches().stream().forEach(deviceId -> {
+ FlowRule flowRuleDirect = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(LOW_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_ACCESS_TYPE)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRuleDirect);
+ });
+ }
+
+ /**
+ * Populates indirect service access rules for ACCESS_TYPE table.
+ * These rules are installed to all devices.
+ *
+ * @param srcRange source range
+ * @param serviceIp service ip
+ * @param outGroups list of output group
+ */
+ private void populateIndirectAccessRule(Ip4Prefix srcRange, Ip4Address serviceIp,
+ Map<DeviceId, GroupId> outGroups) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(srcRange)
+ .matchIPDst(serviceIp.toIpPrefix())
+ .build();
+
+ for (Map.Entry<DeviceId, GroupId> outGroup : outGroups.entrySet()) {
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .group(outGroup.getValue())
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(HIGH_PRIORITY)
+ .forDevice(outGroup.getKey())
+ .forTable(TABLE_ACCESS_TYPE)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+ }
+
+ /**
+ * Populates flow rules for IN_SERVICE table.
+ *
+ * @param inPorts list of inports related to the service for each device
+ * @param outGroups set of output groups
+ */
+ private void populateInServiceRule(Map<DeviceId, Set<PortNumber>> inPorts, Map<DeviceId, GroupId> outGroups) {
+ checkNotNull(inPorts);
+ checkNotNull(outGroups);
+
+ for (Map.Entry<DeviceId, Set<PortNumber>> entry : inPorts.entrySet()) {
+ Set<PortNumber> ports = entry.getValue();
+ DeviceId deviceId = entry.getKey();
+
+ GroupId groupId = outGroups.get(deviceId);
+ if (groupId == null) {
+ continue;
+ }
+
+ ports.stream().forEach(port -> {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(port)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .group(groupId)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_IN_SERVICE)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ });
+ }
+ }
+
+ /**
+ * Populates flow rules for DST_IP table.
+ *
+ * @param deviceId device id
+ * @param inPort in port
+ * @param dstMac mac address
+ * @param dstIp destination ip
+ * @param tunnelId tunnel id
+ * @param tunnelIp tunnel remote ip
+ */
+ private void populateDstIpRule(DeviceId deviceId, PortNumber inPort, MacAddress dstMac,
+ IpAddress dstIp, long tunnelId, IpAddress tunnelIp) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(dstIp.toIpPrefix())
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setEthDst(dstMac)
+ .setOutput(inPort)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_DST_IP)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+
+ for (DeviceId vSwitchId : getVirtualSwitches()) {
+ if (vSwitchId.equals(deviceId)) {
+ continue;
+ }
+
+ ExtensionTreatment tunnelDst = getTunnelDst(vSwitchId, tunnelIp.getIp4Address());
+ if (tunnelDst == null) {
+ continue;
+ }
+
+ treatment = DefaultTrafficTreatment.builder()
+ .setEthDst(dstMac)
+ .setTunnelId(tunnelId)
+ .extension(tunnelDst, vSwitchId)
+ .setOutput(getTunnelPort(vSwitchId))
+ .build();
+
+ flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(vSwitchId)
+ .forTable(TABLE_DST_IP)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+ }
+
+ /**
+ * Populates flow rules for TUNNEL_ID table.
+ *
+ * @param deviceId device id
+ * @param inPort in port
+ * @param mac mac address
+ * @param tunnelId tunnel id
+ */
+ private void populateTunnelInRule(DeviceId deviceId, PortNumber inPort, MacAddress mac, long tunnelId) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchTunnelId(tunnelId)
+ .matchEthDst(mac)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(inPort)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .fromApp(appId)
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(DEFAULT_PRIORITY)
+ .forDevice(deviceId)
+ .forTable(TABLE_TUNNEL_IN)
+ .makePermanent()
+ .build();
+
+ processFlowRule(true, flowRule);
+ }
+
+ /**
+ * Installs or uninstall a given rule.
+ *
+ * @param install true to install, false to uninstall
+ * @param rule rule
+ */
+ private void processFlowRule(boolean install, FlowRule rule) {
+ FlowRuleOperations.Builder oBuilder = FlowRuleOperations.builder();
+ oBuilder = install ? oBuilder.add(rule) : oBuilder.remove(rule);
+
+ flowRuleService.apply(oBuilder.build(new FlowRuleOperationsContext() {
+ @Override
+ public void onError(FlowRuleOperations ops) {
+ log.error(String.format("Failed %s, %s", ops.toString(), rule.toString()));
+ }
+ }));
+ }
+
+ /**
+ * Returns tunnel port of the device.
+ *
+ * @param deviceId device id
+ * @return tunnel port number, or null if no tunnel port exists on a given device
+ */
+ private PortNumber getTunnelPort(DeviceId deviceId) {
+ Port port = deviceService.getPorts(deviceId).stream()
+ .filter(p -> p.annotations().value(PORT_NAME).contains(tunnelType))
+ .findFirst().orElse(null);
+
+ return port == null ? null : port.number();
+ }
+
+ /**
+ * Returns data plane interface port name of a given device.
+ *
+ * @param deviceId device id
+ * @param dpIntf data plane interface port name
+ * @return data plane interface port number, or null if no such port exists
+ */
+ private PortNumber getDpPort(DeviceId deviceId, String dpIntf) {
+ Port port = deviceService.getPorts(deviceId).stream()
+ .filter(p -> p.annotations().value(PORT_NAME).contains(dpIntf) &&
+ p.isEnabled())
+ .findFirst().orElse(null);
+
+ return port == null ? null : port.number();
+ }
+
+ /** Returns data plane interface port number of a given host.
+ *
+ * @param host host
+ * @return port number, or null
+ */
+ private PortNumber getDpPort(Host host) {
+ String portName = host.annotations().value(DATA_PLANE_INTF);
+ return portName == null ? null : getDpPort(host.location().deviceId(), portName);
+ }
+
+ /**
+ * Returns service vlan from a given host.
+ *
+ * @param host host
+ * @return vlan id, or null
+ */
+ private VlanId getServiceVlan(Host host) {
+ String serviceVlan = host.annotations().value(S_TAG);
+ return serviceVlan == null ? null : VlanId.vlanId(Short.parseShort(serviceVlan));
+ }
+
+ /**
+ * Returns the inport from a given flow rule if the rule contains the match of it.
+ *
+ * @param flowRule flow rule
+ * @return port number, or null if the rule doesn't have inport match
+ */
+ private PortNumber getInPort(FlowRule flowRule) {
+ Criterion criterion = flowRule.selector().getCriterion(IN_PORT);
+ if (criterion != null && criterion instanceof PortCriterion) {
+ PortCriterion port = (PortCriterion) criterion;
+ return port.port();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns the destination mac address from a given flow rule if the rule
+ * contains the instruction of it.
+ *
+ * @param flowRule flow rule
+ * @return mac address, or null if the rule doesn't have destination mac instruction
+ */
+ private MacAddress getDstMacFromTreatment(FlowRule flowRule) {
+ Instruction instruction = flowRule.treatment().allInstructions().stream()
+ .filter(inst -> inst instanceof ModEtherInstruction &&
+ ((ModEtherInstruction) inst).subtype().equals(ETH_DST))
+ .findFirst()
+ .orElse(null);
+
+ if (instruction == null) {
+ return null;
+ }
+
+ return ((ModEtherInstruction) instruction).mac();
+ }
+
+ /**
+ * Returns the destination mac address from a given flow rule if the rule
+ * contains the match of it.
+ *
+ * @param flowRule flow rule
+ * @return mac address, or null if the rule doesn't have destination mac match
+ */
+ private MacAddress getDstMacFromSelector(FlowRule flowRule) {
+ Criterion criterion = flowRule.selector().getCriterion(Criterion.Type.ETH_DST);
+ if (criterion != null && criterion instanceof EthCriterion) {
+ EthCriterion eth = (EthCriterion) criterion;
+ return eth.mac();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns the destination IP from a given flow rule if the rule contains
+ * the match of it.
+ *
+ * @param flowRule flow rule
+ * @return ip prefix, or null if the rule doesn't have ip match
+ */
+ private IpPrefix getDstIpFromSelector(FlowRule flowRule) {
+ Criterion criterion = flowRule.selector().getCriterion(IPV4_DST);
+ if (criterion != null && criterion instanceof IPCriterion) {
+ IPCriterion ip = (IPCriterion) criterion;
+ return ip.ip();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns the source IP from a given flow rule if the rule contains
+ * the match of it.
+ *
+ * @param flowRule flow rule
+ * @return ip prefix, or null if the rule doesn't have ip match
+ */
+ private IpPrefix getSrcIpFromSelector(FlowRule flowRule) {
+ Criterion criterion = flowRule.selector().getCriterion(IPV4_SRC);
+ if (criterion != null && criterion instanceof IPCriterion) {
+ IPCriterion ip = (IPCriterion) criterion;
+ return ip.ip();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns the group ID from a given flow rule if the rule contains the
+ * treatment of it.
+ *
+ * @param flowRule flow rule
+ * @return group id, or null if the rule doesn't have group instruction
+ */
+ private GroupId getGroupIdFromTreatment(FlowRule flowRule) {
+ Instruction instruction = flowRule.treatment().allInstructions().stream()
+ .filter(inst -> inst instanceof Instructions.GroupInstruction)
+ .findFirst()
+ .orElse(null);
+
+ if (instruction == null) {
+ return null;
+ }
+
+ return ((Instructions.GroupInstruction) instruction).groupId();
+ }
+
+ /**
+ * Returns the output port number from a given flow rule.
+ *
+ * @param flowRule flow rule
+ * @return port number, or null if the rule does not have output instruction
+ */
+ private PortNumber getOutputFromTreatment(FlowRule flowRule) {
+ Instruction instruction = flowRule.treatment().allInstructions().stream()
+ .filter(inst -> inst instanceof Instructions.OutputInstruction)
+ .findFirst()
+ .orElse(null);
+
+ if (instruction == null) {
+ return null;
+ }
+
+ return ((Instructions.OutputInstruction) instruction).port();
+ }
+
+ /**
+ * Returns if a given flow rule has vlan push instruction or not.
+ *
+ * @param flowRule flow rule
+ * @return true if it includes vlan push, or false
+ */
+ private boolean isVlanPushFromTreatment(FlowRule flowRule) {
+ Instruction instruction = flowRule.treatment().allInstructions().stream()
+ .filter(inst -> inst instanceof L2ModificationInstruction)
+ .filter(inst -> ((L2ModificationInstruction) inst).subtype().equals(VLAN_PUSH))
+ .findAny()
+ .orElse(null);
+
+ return instruction != null;
+ }
+
+ /**
+ * Creates a new group for a given service.
+ *
+ * @param deviceId device id to create a group
+ * @param service cord service
+ * @return group id, or null if it fails to create
+ */
+ private GroupId createServiceGroup(DeviceId deviceId, CordService service) {
+ checkNotNull(service);
+
+ GroupKey groupKey = getGroupKey(service.id());
+ Group group = groupService.getGroup(deviceId, groupKey);
+ GroupId groupId = getGroupId(service.id(), deviceId);
+
+ if (group != null) {
+ log.debug("Group {} is already exist in {}", service.id(), deviceId);
+ return groupId;
+ }
+
+ GroupBuckets buckets = getServiceGroupBuckets(deviceId, service.segmentationId(), service.hosts());
+ GroupDescription groupDescription = new DefaultGroupDescription(
+ deviceId,
+ GroupDescription.Type.SELECT,
+ buckets,
+ groupKey,
+ groupId.id(),
+ appId);
+
+ groupService.addGroup(groupDescription);
+
+ return groupId;
+ }
+
+ /**
+ * Returns group buckets for a given device.
+ *
+ * @param deviceId device id
+ * @param tunnelId tunnel id
+ * @param hosts list of host
+ * @return group buckets
+ */
+ private GroupBuckets getServiceGroupBuckets(DeviceId deviceId, long tunnelId, Map<Host, IpAddress> hosts) {
+ List<GroupBucket> buckets = Lists.newArrayList();
+
+ for (Map.Entry<Host, IpAddress> entry : hosts.entrySet()) {
+ Host host = entry.getKey();
+ Ip4Address remoteIp = entry.getValue().getIp4Address();
+ DeviceId hostDevice = host.location().deviceId();
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
+ .builder()
+ .setEthDst(host.mac());
+
+ if (deviceId.equals(hostDevice)) {
+ tBuilder.setOutput(host.location().port());
+ } else {
+ ExtensionTreatment tunnelDst = getTunnelDst(deviceId, remoteIp);
+ if (tunnelDst == null) {
+ continue;
+ }
+
+ tBuilder.extension(tunnelDst, deviceId)
+ .setTunnelId(tunnelId)
+ .setOutput(getTunnelPort(hostDevice));
+ }
+
+ buckets.add(DefaultGroupBucket.createSelectGroupBucket(tBuilder.build()));
+ }
+
+ return new GroupBuckets(buckets);
+ }
+
+ /**
+ * Returns globally unique group ID.
+ *
+ * @param serviceId service id
+ * @param deviceId device id
+ * @return group id
+ */
+ private GroupId getGroupId(CordServiceId serviceId, DeviceId deviceId) {
+ return new DefaultGroupId(Objects.hash(serviceId, deviceId));
+ }
+
+ /**
+ * Returns group key of a service.
+ *
+ * @param serviceId service id
+ * @return group key
+ */
+ private GroupKey getGroupKey(CordServiceId serviceId) {
+ return new DefaultGroupKey(serviceId.id().getBytes());
+ }
+
+ /**
+ * Returns extension instruction to set tunnel destination.
+ *
+ * @param deviceId device id
+ * @param remoteIp tunnel destination address
+ * @return extension treatment or null if it fails to get instruction
+ */
+ private ExtensionTreatment getTunnelDst(DeviceId deviceId, Ip4Address remoteIp) {
+ try {
+ Driver driver = driverService.getDriver(deviceId);
+ DefaultDriverData driverData = new DefaultDriverData(driver, deviceId);
+ DriverHandler handler = new DefaultDriverHandler(driverData);
+ ExtensionTreatmentResolver resolver = handler.behaviour(ExtensionTreatmentResolver.class);
+
+ ExtensionTreatment treatment =
+ resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type());
+ treatment.setPropertyValue("tunnelDst", remoteIp);
+
+ return treatment;
+ } catch (ItemNotFoundException | UnsupportedOperationException |
+ ExtensionPropertyException e) {
+ log.error("Failed to get extension instruction {}", deviceId);
+ return null;
+ }
+ }
+
+ /**
+ * Returns integration bridges configured in the system.
+ *
+ * @return set of device ids
+ */
+ private Set<DeviceId> getVirtualSwitches() {
+ CordVtnConfig config = configRegistry.getConfig(appId, CordVtnConfig.class);
+ if (config == null) {
+ log.debug("No configuration found for {}", appId.name());
+ return Sets.newHashSet();
+ }
+
+ return config.cordVtnNodes().stream()
+ .map(CordVtnNode::intBrId).collect(Collectors.toSet());
+ }
+}
+
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/RemoteIpCommandUtil.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/RemoteIpCommandUtil.java
new file mode 100644
index 0000000..c09e3a0
--- /dev/null
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/RemoteIpCommandUtil.java
@@ -0,0 +1,243 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cordvtn.impl;
+
+import com.google.common.collect.Sets;
+import com.google.common.io.CharStreams;
+import com.jcraft.jsch.Channel;
+import com.jcraft.jsch.ChannelExec;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.JSchException;
+import com.jcraft.jsch.Session;
+import org.onlab.packet.IpAddress;
+import org.onosproject.cordvtn.api.NetworkAddress;
+import org.onosproject.cordvtn.api.SshAccessInfo;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * {@code RemoteIpCommandUtil} provides methods to help execute Linux IP commands to a remote server.
+ * It opens individual exec channels for each command. User can create a session with {@code connect}
+ * method and then execute a series commands. After done with all commands, the session must be closed
+ * explicitly by calling {@code disconnect}.
+ */
+public final class RemoteIpCommandUtil {
+
+ protected static final Logger log = getLogger(RemoteIpCommandUtil.class);
+
+ private static final String STRICT_HOST_CHECKING = "StrictHostKeyChecking";
+ private static final String DEFAULT_STRICT_HOST_CHECKING = "no";
+ private static final int DEFAULT_SESSION_TIMEOUT = 60000; // milliseconds
+
+ private static final String IP_PATTERN = "^([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+ "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+ "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+ "([01]?\\d\\d?|2[0-4]\\d|25[0-5])$";
+
+ private static final String IP_ADDR_SHOW = "sudo ip addr show %s";
+ private static final String IP_ADDR_FLUSH = "sudo ip addr flush %s";
+ private static final String IP_ADDR_ADD = "sudo ip addr add %s dev %s";
+ private static final String IP_ADDR_DELETE = "sudo ip addr delete %s dev %s";
+ private static final String IP_LINK_SHOW = "sudo ip link show %s";
+ private static final String IP_LINK_UP = "sudo ip link set %s up";
+
+ /**
+ * Default constructor.
+ */
+ private RemoteIpCommandUtil() {
+ }
+
+ /**
+ * Adds a given IP address to a given device.
+ *
+ * @param session ssh connection
+ * @param ip network address
+ * @param device device name to assign the ip address
+ * @return true if the command succeeds, or false
+ */
+ public static boolean addIp(Session session, NetworkAddress ip, String device) {
+ if (session == null || !session.isConnected()) {
+ return false;
+ }
+
+ executeCommand(session, String.format(IP_ADDR_ADD, ip.cidr(), device));
+ Set<IpAddress> result = getCurrentIps(session, device);
+ return result.contains(ip.ip());
+ }
+
+ /**
+ * Removes the IP address from a given device.
+ *
+ * @param session ssh connection
+ * @param ip ip address
+ * @param device device name
+ * @return true if the command succeeds, or false
+ */
+ public static boolean deleteIp(Session session, IpAddress ip, String device) {
+ if (session == null || !session.isConnected()) {
+ return false;
+ }
+
+ executeCommand(session, String.format(IP_ADDR_DELETE, ip, device));
+ Set<IpAddress> result = getCurrentIps(session, device);
+ return !result.contains(ip);
+ }
+
+ /**
+ * Removes all IP address on a given device.
+ *
+ * @param session ssh connection
+ * @param device device name
+ * @return true if the command succeeds, or false
+ */
+ public static boolean flushIp(Session session, String device) {
+ if (session == null || !session.isConnected()) {
+ return false;
+ }
+
+ executeCommand(session, String.format(IP_ADDR_FLUSH, device));
+ return getCurrentIps(session, device).isEmpty();
+ }
+
+ /**
+ * Returns a set of IP address that a given device has.
+ *
+ * @param session ssh connection
+ * @param device device name
+ * @return set of IP prefix or empty set
+ */
+ public static Set<IpAddress> getCurrentIps(Session session, String device) {
+ if (session == null || !session.isConnected()) {
+ return Sets.newHashSet();
+ }
+
+ String output = executeCommand(session, String.format(IP_ADDR_SHOW, device));
+ Set<IpAddress> result = Pattern.compile(" |/")
+ .splitAsStream(output)
+ .filter(s -> s.matches(IP_PATTERN))
+ .map(IpAddress::valueOf)
+ .collect(Collectors.toSet());
+
+ return result;
+ }
+
+ /**
+ * Sets link state up for a given device.
+ *
+ * @param session ssh connection
+ * @param device device name
+ * @return true if the command succeeds, or false
+ */
+ public static boolean setInterfaceUp(Session session, String device) {
+ if (session == null || !session.isConnected()) {
+ return false;
+ }
+
+ executeCommand(session, String.format(IP_LINK_UP, device));
+ return isInterfaceUp(session, device);
+ }
+
+ /**
+ * Checks if a given interface is up or not.
+ *
+ * @param session ssh connection
+ * @param device device name
+ * @return true if the interface is up, or false
+ */
+ public static boolean isInterfaceUp(Session session, String device) {
+ if (session == null || !session.isConnected()) {
+ return false;
+ }
+
+ String output = executeCommand(session, String.format(IP_LINK_SHOW, device));
+ return output != null && output.contains("UP");
+ }
+
+ /**
+ * Creates a new session with a given access information.
+ *
+ * @param sshInfo information to ssh to the remove server
+ * @return ssh session, or null
+ */
+ public static Session connect(SshAccessInfo sshInfo) {
+ try {
+ JSch jsch = new JSch();
+ jsch.addIdentity(sshInfo.privateKey());
+
+ Session session = jsch.getSession(sshInfo.user(),
+ sshInfo.remoteIp().toString(),
+ sshInfo.port().toInt());
+ session.setConfig(STRICT_HOST_CHECKING, DEFAULT_STRICT_HOST_CHECKING);
+ session.connect(DEFAULT_SESSION_TIMEOUT);
+
+ return session;
+ } catch (JSchException e) {
+ log.debug("Failed to connect to {} due to {}", sshInfo.toString(), e.toString());
+ return null;
+ }
+ }
+
+ /**
+ * Closes a connection.
+ *
+ * @param session session
+ */
+ public static void disconnect(Session session) {
+ if (session.isConnected()) {
+ session.disconnect();
+ }
+ }
+
+ /**
+ * Executes a given command. It opens exec channel for the command and closes
+ * the channel when it's done.
+ *
+ * @param session ssh connection to a remote server
+ * @param command command to execute
+ * @return command output string if the command succeeds, or null
+ */
+ private static String executeCommand(Session session, String command) {
+ if (session == null || !session.isConnected()) {
+ return null;
+ }
+
+ log.trace("Execute command {} to {}", command, session.getHost());
+
+ try {
+ Channel channel = session.openChannel("exec");
+ ((ChannelExec) channel).setCommand(command);
+ channel.setInputStream(null);
+ InputStream output = channel.getInputStream();
+
+ channel.connect();
+ String result = CharStreams.toString(new InputStreamReader(output));
+ channel.disconnect();
+
+ return result;
+ } catch (JSchException | IOException e) {
+ log.debug("Failed to execute command {} due to {}", command, e.toString());
+ return null;
+ }
+ }
+}
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/package-info.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/package-info.java
new file mode 100644
index 0000000..fcc52ff
--- /dev/null
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/impl/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementation for CORD VTN application.
+ */
+package org.onosproject.cordvtn.impl;
\ No newline at end of file