Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018-present Open Networking Foundation |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package org.onosproject.openstacknetworking.impl; |
| 17 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 18 | import org.onosproject.cluster.ClusterService; |
| 19 | import org.onosproject.cluster.LeadershipService; |
| 20 | import org.onosproject.cluster.NodeId; |
| 21 | import org.onosproject.core.ApplicationId; |
| 22 | import org.onosproject.core.CoreService; |
| 23 | import org.onosproject.mastership.MastershipService; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 24 | import org.onosproject.net.Device; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 25 | import org.onosproject.net.DeviceId; |
| 26 | import org.onosproject.net.device.DeviceService; |
| 27 | import org.onosproject.openstacknetworking.api.InstancePort; |
| 28 | import org.onosproject.openstacknetworking.api.InstancePortService; |
| 29 | import org.onosproject.openstacknetworking.api.OpenstackNetworkEvent; |
| 30 | import org.onosproject.openstacknetworking.api.OpenstackNetworkListener; |
| 31 | import org.onosproject.openstacknetworking.api.OpenstackNetworkService; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 32 | import org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 33 | import org.onosproject.openstacknode.api.OpenstackNode; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 34 | import org.onosproject.openstacknode.api.OpenstackNodeEvent; |
| 35 | import org.onosproject.openstacknode.api.OpenstackNodeListener; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 36 | import org.onosproject.openstacknode.api.OpenstackNodeService; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 37 | import org.openstack4j.model.network.Port; |
| 38 | import org.openstack4j.model.network.State; |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 39 | import org.osgi.service.component.annotations.Activate; |
| 40 | import org.osgi.service.component.annotations.Component; |
| 41 | import org.osgi.service.component.annotations.Deactivate; |
| 42 | import org.osgi.service.component.annotations.Reference; |
| 43 | import org.osgi.service.component.annotations.ReferenceCardinality; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 44 | import org.slf4j.Logger; |
| 45 | import org.slf4j.LoggerFactory; |
| 46 | |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 47 | import java.util.List; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 48 | import java.util.Objects; |
| 49 | import java.util.Optional; |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 50 | import java.util.concurrent.ExecutorService; |
| 51 | import java.util.concurrent.Executors; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 52 | import java.util.stream.Collectors; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 53 | |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 54 | import static java.lang.Thread.sleep; |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 55 | import static org.onlab.util.Tools.groupedThreads; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 56 | import static org.onosproject.openstacknetworking.api.Constants.DIRECT; |
| 57 | import static org.onosproject.openstacknetworking.api.Constants.OPENSTACK_NETWORKING_APP_ID; |
Daniel Park | ec9d113 | 2018-08-19 11:18:03 +0900 | [diff] [blame] | 58 | import static org.onosproject.openstacknetworking.api.Constants.UNSUPPORTED_VENDOR; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 59 | import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.getIntfNameFromPciAddress; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 60 | import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.hasIntfAleadyInDevice; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 61 | import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.COMPUTE; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 62 | import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.CONTROLLER; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 63 | |
| 64 | @Component(immediate = true) |
| 65 | public final class OpenStackSwitchingDirectPortProvider { |
| 66 | private final Logger log = LoggerFactory.getLogger(getClass()); |
| 67 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 68 | private static final String UNBOUND = "unbound"; |
| 69 | private static final String PORT_NAME = "portName"; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 70 | private static final long SLEEP_MS = 3000; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 71 | |
Ray Milkey | d84f89b | 2018-08-17 14:54:17 -0700 | [diff] [blame] | 72 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 73 | protected CoreService coreService; |
| 74 | |
Ray Milkey | d84f89b | 2018-08-17 14:54:17 -0700 | [diff] [blame] | 75 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 76 | protected OpenstackNetworkService osNetworkService; |
| 77 | |
Ray Milkey | d84f89b | 2018-08-17 14:54:17 -0700 | [diff] [blame] | 78 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 79 | protected OpenstackNodeService osNodeService; |
| 80 | |
Ray Milkey | d84f89b | 2018-08-17 14:54:17 -0700 | [diff] [blame] | 81 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 82 | protected LeadershipService leadershipService; |
| 83 | |
Ray Milkey | d84f89b | 2018-08-17 14:54:17 -0700 | [diff] [blame] | 84 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 85 | protected ClusterService clusterService; |
| 86 | |
Ray Milkey | d84f89b | 2018-08-17 14:54:17 -0700 | [diff] [blame] | 87 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 88 | protected DeviceService deviceService; |
| 89 | |
Ray Milkey | d84f89b | 2018-08-17 14:54:17 -0700 | [diff] [blame] | 90 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 91 | protected InstancePortService instancePortService; |
| 92 | |
Ray Milkey | d84f89b | 2018-08-17 14:54:17 -0700 | [diff] [blame] | 93 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 94 | protected MastershipService mastershipService; |
| 95 | |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 96 | private final ExecutorService executor = |
| 97 | Executors.newSingleThreadExecutor(groupedThreads(this.getClass().getSimpleName(), "direct-port-event")); |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 98 | private final OpenstackNetworkListener openstackNetworkListener = new InternalOpenstackNetworkListener(); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 99 | private final InternalOpenstackNodeListener internalNodeListener = new InternalOpenstackNodeListener(); |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 100 | |
| 101 | private NodeId localNodeId; |
| 102 | private ApplicationId appId; |
| 103 | |
| 104 | @Activate |
| 105 | void activate() { |
| 106 | appId = coreService.registerApplication(OPENSTACK_NETWORKING_APP_ID); |
| 107 | localNodeId = clusterService.getLocalNode().id(); |
| 108 | leadershipService.runForLeadership(appId.name()); |
| 109 | osNetworkService.addListener(openstackNetworkListener); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 110 | osNodeService.addListener(internalNodeListener); |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 111 | |
| 112 | log.info("Started"); |
| 113 | } |
| 114 | |
| 115 | @Deactivate |
| 116 | void deactivate() { |
| 117 | leadershipService.withdraw(appId.name()); |
| 118 | osNetworkService.removeListener(openstackNetworkListener); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 119 | osNodeService.removeListener(internalNodeListener); |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 120 | executor.shutdown(); |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 121 | |
| 122 | log.info("Stopped"); |
| 123 | } |
| 124 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 125 | private class InternalOpenstackNetworkListener implements OpenstackNetworkListener { |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 126 | private boolean isRelevantHelper() { |
| 127 | return Objects.equals(localNodeId, leadershipService.getLeader(appId.name())); |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 128 | } |
| 129 | |
| 130 | @Override |
| 131 | public void event(OpenstackNetworkEvent event) { |
| 132 | switch (event.type()) { |
| 133 | case OPENSTACK_PORT_UPDATED: |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 134 | |
| 135 | executor.execute(() -> { |
| 136 | |
| 137 | if (!isRelevantHelper()) { |
| 138 | return; |
| 139 | } |
| 140 | |
| 141 | if (event.port().getState() == State.DOWN) { |
| 142 | processPortRemoved(event.port()); |
| 143 | } else { |
| 144 | processPortAdded(event.port()); |
| 145 | } |
| 146 | }); |
| 147 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 148 | break; |
| 149 | case OPENSTACK_PORT_REMOVED: |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 150 | |
| 151 | executor.execute(() -> { |
| 152 | |
| 153 | if (!isRelevantHelper()) { |
| 154 | return; |
| 155 | } |
| 156 | |
| 157 | processPortRemoved(event.port()); |
| 158 | }); |
| 159 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 160 | break; |
| 161 | default: |
| 162 | break; |
| 163 | |
| 164 | } |
| 165 | } |
Daniel Park | ff178ba | 2018-11-23 15:57:24 +0900 | [diff] [blame^] | 166 | |
| 167 | private void processPortAdded(Port port) { |
| 168 | if (!port.getvNicType().equals(DIRECT)) { |
| 169 | return; |
| 170 | } else if (!port.isAdminStateUp() || port.getVifType().equals(UNBOUND)) { |
| 171 | log.trace("processPortAdded skipped because of status: {}, adminStateUp: {}, vifType: {}", |
| 172 | port.getState(), port.isAdminStateUp(), port.getVifType()); |
| 173 | return; |
| 174 | } else { |
| 175 | Optional<OpenstackNode> osNode = osNodeService.completeNodes(COMPUTE).stream() |
| 176 | .filter(node -> node.hostname().equals(port.getHostId())) |
| 177 | .findAny(); |
| 178 | if (!osNode.isPresent()) { |
| 179 | log.error("processPortAdded failed because openstackNode doesn't exist that matches hostname {}", |
| 180 | port.getHostId()); |
| 181 | return; |
| 182 | } |
| 183 | log.trace("Retrieved openstackNode: {}", osNode.get().toString()); |
| 184 | |
| 185 | String intfName = getIntfNameFromPciAddress(port); |
| 186 | if (intfName == null) { |
| 187 | log.error("Failed to execute processPortAdded because of null interface name"); |
| 188 | return; |
| 189 | } else if (intfName.equals(UNSUPPORTED_VENDOR)) { |
| 190 | return; |
| 191 | } |
| 192 | log.trace("Retrieved interface name: {}", intfName); |
| 193 | |
| 194 | try { |
| 195 | //If the VF port has been already added to the device for some reason, we remove it first, |
| 196 | //and then add VF so that other handlers run their logic. |
| 197 | if (hasIntfAleadyInDevice(osNode.get().intgBridge(), |
| 198 | intfName, deviceService)) { |
| 199 | log.trace("Device {} has already has VF interface {}, so remove first.", |
| 200 | osNode.get().intgBridge(), |
| 201 | intfName); |
| 202 | osNodeService.removeVfPort(osNode.get(), intfName); |
| 203 | //we wait 3000ms because the ovsdb client can't deal with removal/add at the same time. |
| 204 | sleep(SLEEP_MS); |
| 205 | } |
| 206 | } catch (InterruptedException e) { |
| 207 | log.error("Exception occurred because of {}", e.toString()); |
| 208 | } |
| 209 | |
| 210 | osNodeService.addVfPort(osNode.get(), intfName); |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | private void processPortRemoved(Port port) { |
| 215 | if (!port.getvNicType().equals(DIRECT)) { |
| 216 | return; |
| 217 | } else if (instancePortService.instancePort(port.getId()) == null) { |
| 218 | log.trace("processPortRemoved skipped because no instance port exist for portId: {}", port.getId()); |
| 219 | return; |
| 220 | } else { |
| 221 | InstancePort instancePort = instancePortService.instancePort(port.getId()); |
| 222 | if (instancePort == null) { |
| 223 | return; |
| 224 | } |
| 225 | DeviceId deviceId = instancePort.deviceId(); |
| 226 | if (deviceId == null) { |
| 227 | return; |
| 228 | } |
| 229 | OpenstackNode osNode = osNodeService.node(deviceId); |
| 230 | if (osNode == null) { |
| 231 | return; |
| 232 | } |
| 233 | |
| 234 | Optional<org.onosproject.net.Port> removedPort = deviceService.getPorts(deviceId).stream() |
| 235 | .filter(p -> Objects.equals(p.number(), instancePort.portNumber())) |
| 236 | .findAny(); |
| 237 | |
| 238 | if (!removedPort.isPresent()) { |
| 239 | log.error("Failed to execute processPortAdded because port number doesn't exist"); |
| 240 | return; |
| 241 | } |
| 242 | |
| 243 | String intfName = removedPort.get().annotations().value(PORT_NAME); |
| 244 | |
| 245 | if (intfName == null) { |
| 246 | log.error("Failed to execute processPortAdded because of null interface name"); |
| 247 | return; |
| 248 | } |
| 249 | log.trace("Retrieved interface name: {}", intfName); |
| 250 | |
| 251 | osNodeService.removeVfPort(osNode, intfName); |
| 252 | } |
| 253 | } |
| 254 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 255 | } |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 256 | |
| 257 | private class InternalOpenstackNodeListener implements OpenstackNodeListener { |
| 258 | |
| 259 | @Override |
| 260 | public boolean isRelevant(OpenstackNodeEvent event) { |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 261 | return event.subject().type() != CONTROLLER; |
| 262 | } |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 263 | |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 264 | private boolean isRelevantHelper(OpenstackNodeEvent event) { |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 265 | // do not allow to proceed without mastership |
| 266 | Device device = deviceService.getDevice(event.subject().intgBridge()); |
| 267 | if (device == null) { |
| 268 | return false; |
| 269 | } |
| 270 | return mastershipService.isLocalMaster(device.id()); |
| 271 | } |
| 272 | |
| 273 | @Override |
| 274 | public void event(OpenstackNodeEvent event) { |
| 275 | OpenstackNode osNode = event.subject(); |
| 276 | |
| 277 | switch (event.type()) { |
| 278 | case OPENSTACK_NODE_COMPLETE: |
| 279 | log.info("COMPLETE node {} is detected", osNode.hostname()); |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 280 | |
| 281 | executor.execute(() -> { |
| 282 | |
| 283 | if (!isRelevantHelper(event)) { |
| 284 | return; |
| 285 | } |
| 286 | |
| 287 | processComputeState(event.subject()); |
| 288 | }); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 289 | |
| 290 | break; |
| 291 | case OPENSTACK_NODE_INCOMPLETE: |
| 292 | case OPENSTACK_NODE_CREATED: |
| 293 | case OPENSTACK_NODE_UPDATED: |
| 294 | case OPENSTACK_NODE_REMOVED: |
| 295 | // not reacts to the events other than complete and incomplete states |
| 296 | break; |
| 297 | default: |
| 298 | break; |
| 299 | } |
| 300 | } |
| 301 | |
| 302 | private void processComputeState(OpenstackNode node) { |
| 303 | List<Port> ports = osNetworkService.ports().stream() |
| 304 | .filter(port -> port.getvNicType().equals(DIRECT)) |
| 305 | .filter(port -> !port.getVifType().equals(UNBOUND)) |
| 306 | .filter(port -> port.getHostId().equals(node.hostname())) |
| 307 | .collect(Collectors.toList()); |
| 308 | |
Jian Li | 34220ea | 2018-11-14 01:30:24 +0900 | [diff] [blame] | 309 | ports.forEach(port -> addIntfToDevice(node, port)); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 310 | } |
| 311 | |
| 312 | private void addIntfToDevice(OpenstackNode node, Port port) { |
| 313 | String intfName = OpenstackNetworkingUtil.getIntfNameFromPciAddress(port); |
| 314 | if (intfName == null) { |
| 315 | log.error("Failed to retrieve interface name from a port {}", port.getId()); |
Daniel Park | c3e0c60 | 2018-08-30 21:27:25 +0900 | [diff] [blame] | 316 | } else if (intfName.equals(UNSUPPORTED_VENDOR)) { |
| 317 | log.warn("Failed to retrieve interface name from a port {} because of unsupported ovs-based sr-iov"); |
| 318 | return; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 319 | } |
| 320 | |
| 321 | if (!hasIntfAleadyInDevice(node.intgBridge(), intfName, deviceService)) { |
| 322 | log.debug("Port {} is bound to the interface {} but not added in the bridge {}. Adding it..", |
| 323 | port.getId(), |
| 324 | intfName, |
| 325 | node.intgBridge()); |
| 326 | osNodeService.addVfPort(node, intfName); |
| 327 | } |
| 328 | } |
| 329 | } |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 330 | } |