[ONOS-6060] Merge scalablegateway app to openstacknode app
Change-Id: I8e152a943f575c2fc0305ae572b5e0150a699d30
diff --git a/apps/openstacknode/src/main/java/org/onosproject/openstacknode/OpenstackNodeManager.java b/apps/openstacknode/src/main/java/org/onosproject/openstacknode/OpenstackNodeManager.java
index 5276c58..0bf78bf 100644
--- a/apps/openstacknode/src/main/java/org/onosproject/openstacknode/OpenstackNodeManager.java
+++ b/apps/openstacknode/src/main/java/org/onosproject/openstacknode/OpenstackNodeManager.java
@@ -15,6 +15,8 @@
*/
package org.onosproject.openstacknode;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
@@ -35,6 +37,7 @@
import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
+import org.onosproject.core.GroupId;
import org.onosproject.event.ListenerRegistry;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
@@ -60,6 +63,10 @@
import org.onosproject.net.device.DeviceEvent;
import org.onosproject.net.device.DeviceListener;
import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupKey;
+import org.onosproject.net.group.GroupService;
import org.onosproject.openstacknode.OpenstackNodeEvent.NodeState;
import org.onosproject.ovsdb.controller.OvsdbClientService;
import org.onosproject.ovsdb.controller.OvsdbController;
@@ -113,6 +120,7 @@
private static final int DPID_BEGIN = 3;
private static final String APP_ID = "org.onosproject.openstacknode";
+
private static final Class<OpenstackNodeConfig> CONFIG_CLASS = OpenstackNodeConfig.class;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
@@ -139,6 +147,12 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected LeadershipService leadershipService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DriverService driverService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected GroupService groupService;
+
@Property(name = OVSDB_PORT, intValue = DEFAULT_OVSDB_PORT,
label = "OVSDB server listen port")
private int ovsdbPort = DEFAULT_OVSDB_PORT;
@@ -164,6 +178,7 @@
private ConsistentMap<String, OpenstackNode> nodeStore;
+ private SelectGroupHandler selectGroupHandler;
private ApplicationId appId;
private NodeId localNodeId;
@@ -187,6 +202,8 @@
configRegistry.addListener(configListener);
componentConfigService.registerProperties(getClass());
+ selectGroupHandler = new SelectGroupHandler(groupService, deviceService, driverService, appId);
+
readConfiguration();
log.info("Started");
}
@@ -275,12 +292,25 @@
@Override
public void processCompleteState(OpenstackNode node) {
process(new OpenstackNodeEvent(COMPLETE, node));
+ switch (node.type()) {
+ case COMPUTE:
+ selectGroupHandler.createGatewayGroup(node.intBridge(), gatewayNodes());
+ break;
+ case GATEWAY:
+ updateGatewayGroup(node, true);
+ break;
+ default:
+ break;
+ }
log.info("Finished init {}", node.hostname());
}
@Override
public void processIncompleteState(OpenstackNode node) {
process(new OpenstackNodeEvent(INCOMPLETE, node));
+ if (node.type().equals(NodeType.GATEWAY)) {
+ updateGatewayGroup(node, false);
+ }
}
@Override
@@ -331,6 +361,65 @@
.map(Port::number).findFirst();
}
+ @Override
+ public OpenstackNode gatewayNode(DeviceId deviceId) {
+ OpenstackNode gatewayNode = nodeByDeviceId(deviceId);
+ if (gatewayNode == null) {
+ log.warn("Gateway with device ID {} does not exist");
+ return null;
+ }
+ return gatewayNode;
+ }
+
+ @Override
+ public synchronized GroupId gatewayGroupId(DeviceId srcDeviceId) {
+ GroupKey groupKey = selectGroupHandler.getGroupKey(srcDeviceId);
+ Group group = groupService.getGroup(srcDeviceId, groupKey);
+ if (group == null) {
+ log.info("Created gateway group for {}", srcDeviceId);
+ return selectGroupHandler.createGatewayGroup(srcDeviceId, gatewayNodes());
+ } else {
+ return group.id();
+ }
+ }
+
+ @Override
+ public List<OpenstackNode> gatewayNodes() {
+ return nodeStore.values()
+ .stream()
+ .map(Versioned::value)
+ .filter(node -> node.type().equals(NodeType.GATEWAY))
+ .filter(node -> node.state().equals(COMPLETE))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<DeviceId> gatewayDeviceIds() {
+ List<DeviceId> deviceIdList = Lists.newArrayList();
+
+ nodeStore.values()
+ .stream()
+ .map(Versioned::value)
+ .filter(node -> node.type().equals(NodeType.GATEWAY))
+ .filter(node -> node.state().equals(COMPLETE))
+ .forEach(node -> deviceIdList.add(node.intBridge()));
+ return deviceIdList;
+ }
+
+ private void updateGatewayGroup(OpenstackNode gatewayNode, boolean isInsert) {
+ nodeStore.values()
+ .stream()
+ .map(Versioned::value)
+ .filter(node -> node.type().equals(NodeType.COMPUTE))
+ .filter(node -> node.state().equals(COMPLETE))
+ .forEach(node -> {
+ selectGroupHandler.updateGatewayGroupBuckets(node.intBridge(),
+ ImmutableList.of(gatewayNode),
+ isInsert);
+ log.trace("Updated gateway group on {}", node.intBridge());
+ });
+ }
+
private void initNode(OpenstackNode node) {
NodeState state = node.state();
state.process(this, node);
@@ -675,6 +764,7 @@
eventExecutor.execute(() -> handler.connected(device));
} else {
eventExecutor.execute(() -> handler.disconnected(device));
+ log.warn("OpenstackNode with device ID {} is disconnected", device.id());
}
break;
default:
diff --git a/apps/openstacknode/src/main/java/org/onosproject/openstacknode/OpenstackNodeService.java b/apps/openstacknode/src/main/java/org/onosproject/openstacknode/OpenstackNodeService.java
index f7c16ba..0fb6e77 100644
--- a/apps/openstacknode/src/main/java/org/onosproject/openstacknode/OpenstackNodeService.java
+++ b/apps/openstacknode/src/main/java/org/onosproject/openstacknode/OpenstackNodeService.java
@@ -16,6 +16,7 @@
package org.onosproject.openstacknode;
import org.onlab.packet.IpAddress;
+import org.onosproject.core.GroupId;
import org.onosproject.event.ListenerService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
@@ -127,4 +128,34 @@
* @return port number; or empty value
*/
Optional<PortNumber> externalPort(DeviceId intBridgeId);
+ /**
+ * Returns gateway node with the given device identifier.
+ *
+ * @param deviceId The gateway node deviceId
+ * @return The gateway node information
+ */
+ OpenstackNode gatewayNode(DeviceId deviceId);
+
+ /**
+ * Returns group id for gateway load balance.
+ * If the group does not exist in the supplied source device, creates one.
+ *
+ * @param srcDeviceId source device id
+ * @return The group id
+ */
+ GroupId gatewayGroupId(DeviceId srcDeviceId);
+
+ /**
+ * Returns the list of gateway node information with the given device identifier.
+ *
+ * @return The list of gateway node information
+ */
+ List<OpenstackNode> gatewayNodes();
+
+ /**
+ * Returns the list of gateway`s device identifiers.
+ *
+ * @return The list of device identifier]
+ */
+ List<DeviceId> gatewayDeviceIds();
}
diff --git a/apps/openstacknode/src/main/java/org/onosproject/openstacknode/SelectGroupHandler.java b/apps/openstacknode/src/main/java/org/onosproject/openstacknode/SelectGroupHandler.java
new file mode 100644
index 0000000..df963db
--- /dev/null
+++ b/apps/openstacknode/src/main/java/org/onosproject/openstacknode/SelectGroupHandler.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright 2017-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.openstacknode;
+
+import com.google.common.collect.Lists;
+import org.onlab.packet.Ip4Address;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.GroupId;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DefaultDriverData;
+import org.onosproject.net.driver.DefaultDriverHandler;
+import org.onosproject.net.driver.Driver;
+import org.onosproject.net.driver.DriverHandler;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionPropertyException;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatmentType;
+import org.onosproject.net.group.DefaultGroupDescription;
+import org.onosproject.net.group.DefaultGroupKey;
+import org.onosproject.net.group.GroupBucket;
+import org.onosproject.net.group.GroupBuckets;
+import org.onosproject.net.group.GroupDescription;
+import org.onosproject.net.group.GroupKey;
+import org.onosproject.net.group.GroupService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+import static org.onosproject.net.AnnotationKeys.PORT_NAME;
+import static org.onosproject.net.group.DefaultGroupBucket.createSelectGroupBucket;
+
+/**
+ * Handles group generation request from OpenstackNode.
+ */
+public class SelectGroupHandler {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private static final String TUNNEL_DESTINATION = "tunnelDst";
+ private static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
+
+ private final GroupService groupService;
+ private final DeviceService deviceService;
+ private final DriverService driverService;
+ private final ApplicationId appId;
+
+ /**
+ * Default constructor.
+ *
+ * @param targetGroupService group service
+ * @param targetDeviceService device service
+ * @param targetDriverService driver service
+ * @param appId application id for group service
+ */
+ public SelectGroupHandler(GroupService targetGroupService, DeviceService targetDeviceService,
+ DriverService targetDriverService, ApplicationId appId) {
+ groupService = targetGroupService;
+ deviceService = targetDeviceService;
+ driverService = targetDriverService;
+ this.appId = appId;
+ }
+
+ /**
+ * Creates select type group description according to given deviceId.
+ *
+ * @param srcDeviceId target device id for group description
+ * @param gatewayNodeList gateway node list for bucket action
+ * @return created select type group description
+ */
+ public GroupId createGatewayGroup(DeviceId srcDeviceId, List<OpenstackNode> gatewayNodeList) {
+ List<GroupBucket> bucketList = generateBucketsForSelectGroup(srcDeviceId, gatewayNodeList);
+ GroupId groupId = getGroupId(srcDeviceId);
+ GroupDescription groupDescription = new DefaultGroupDescription(
+ srcDeviceId,
+ GroupDescription.Type.SELECT,
+ new GroupBuckets(bucketList),
+ getGroupKey(srcDeviceId),
+ groupId.id(),
+ appId);
+
+ groupService.addGroup(groupDescription);
+ return groupId;
+ }
+
+ /**
+ * Returns unique group key with supplied source device ID as a hash.
+ *
+ * @param srcDeviceId source device id
+ * @return group key
+ */
+ public GroupKey getGroupKey(DeviceId srcDeviceId) {
+ return new DefaultGroupKey(srcDeviceId.toString().getBytes());
+ }
+
+ private GroupId getGroupId(DeviceId srcDeviceId) {
+ return new GroupId(srcDeviceId.toString().hashCode());
+ }
+
+ /**
+ * Updates groupBuckets in select type group.
+ *
+ * @param deviceId target device id to update the group
+ * @param gatewayNodeList updated gateway node list for bucket action
+ * @param isInsert update type(add or remove)
+ */
+ public void updateGatewayGroupBuckets(DeviceId deviceId,
+ List<OpenstackNode> gatewayNodeList,
+ boolean isInsert) {
+ List<GroupBucket> bucketList = generateBucketsForSelectGroup(deviceId, gatewayNodeList);
+ GroupKey groupKey = getGroupKey(deviceId);
+ if (groupService.getGroup(deviceId, groupKey) == null) {
+ log.error("There's no group in compute node {}", deviceId);
+ return;
+ }
+
+ if (isInsert) {
+ groupService.addBucketsToGroup(
+ deviceId,
+ groupKey,
+ new GroupBuckets(bucketList),
+ groupKey, appId);
+ } else {
+ groupService.removeBucketsFromGroup(
+ deviceId,
+ groupKey,
+ new GroupBuckets(bucketList),
+ groupKey, appId);
+ }
+ }
+
+ private List<GroupBucket> generateBucketsForSelectGroup(DeviceId deviceId, List<OpenstackNode> gatewayNodeList) {
+ List<GroupBucket> bucketList = Lists.newArrayList();
+ gatewayNodeList.forEach(node -> {
+ TrafficTreatment tBuilder = DefaultTrafficTreatment.builder()
+ .extension(buildNiciraExtenstion(deviceId, node.dataIp().get().getIp4Address()), deviceId)
+ .setOutput(getTunnelPort(deviceId))
+ .build();
+ bucketList.add(createSelectGroupBucket(tBuilder));
+ });
+ return bucketList;
+ }
+
+ /**
+ * Builds Nicira extension for tagging remoteIp of vxlan.
+ *
+ * @param id device id of vxlan source device
+ * @param hostIp remote ip of vxlan destination device
+ * @return NiciraExtension Treatment
+ */
+ private ExtensionTreatment buildNiciraExtenstion(DeviceId id, Ip4Address hostIp) {
+ Driver driver = driverService.getDriver(id);
+ DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, id));
+ ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
+
+ ExtensionTreatment extensionInstruction =
+ resolver.getExtensionInstruction(
+ ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST.type());
+
+ try {
+ extensionInstruction.setPropertyValue(TUNNEL_DESTINATION, hostIp);
+ } catch (ExtensionPropertyException e) {
+ log.error("Error setting Nicira extension setting {}", e);
+ }
+
+ return extensionInstruction;
+ }
+
+ /**
+ * Returns port number of vxlan tunnel.
+ *
+ * @param deviceId target Device Id
+ * @return portNumber
+ */
+ private PortNumber getTunnelPort(DeviceId deviceId) {
+ Port port = deviceService.getPorts(deviceId).stream()
+ .filter(p -> p.annotations().value(PORT_NAME).equals(PORTNAME_PREFIX_TUNNEL))
+ .findAny().orElse(null);
+
+ if (port == null) {
+ log.error("No TunnelPort was created.");
+ return null;
+ }
+ return port.number();
+
+ }
+}