Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018-present Open Networking Foundation |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package org.onosproject.openstacknetworking.impl; |
| 17 | |
| 18 | import org.apache.felix.scr.annotations.Activate; |
| 19 | import org.apache.felix.scr.annotations.Component; |
| 20 | import org.apache.felix.scr.annotations.Deactivate; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 21 | import org.apache.felix.scr.annotations.Reference; |
| 22 | import org.apache.felix.scr.annotations.ReferenceCardinality; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 23 | import org.onosproject.cluster.ClusterService; |
| 24 | import org.onosproject.cluster.LeadershipService; |
| 25 | import org.onosproject.cluster.NodeId; |
| 26 | import org.onosproject.core.ApplicationId; |
| 27 | import org.onosproject.core.CoreService; |
| 28 | import org.onosproject.mastership.MastershipService; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 29 | import org.onosproject.net.Device; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 30 | import org.onosproject.net.DeviceId; |
| 31 | import org.onosproject.net.device.DeviceService; |
| 32 | import org.onosproject.openstacknetworking.api.InstancePort; |
| 33 | import org.onosproject.openstacknetworking.api.InstancePortService; |
| 34 | import org.onosproject.openstacknetworking.api.OpenstackNetworkEvent; |
| 35 | import org.onosproject.openstacknetworking.api.OpenstackNetworkListener; |
| 36 | import org.onosproject.openstacknetworking.api.OpenstackNetworkService; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 37 | import org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 38 | import org.onosproject.openstacknode.api.OpenstackNode; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 39 | import org.onosproject.openstacknode.api.OpenstackNodeEvent; |
| 40 | import org.onosproject.openstacknode.api.OpenstackNodeListener; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 41 | import org.onosproject.openstacknode.api.OpenstackNodeService; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 42 | import org.openstack4j.model.network.Port; |
| 43 | import org.openstack4j.model.network.State; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 44 | import org.slf4j.Logger; |
| 45 | import org.slf4j.LoggerFactory; |
| 46 | |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 47 | import java.util.List; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 48 | import java.util.Objects; |
| 49 | import java.util.Optional; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 50 | import java.util.stream.Collectors; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 51 | |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 52 | import static java.lang.Thread.sleep; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 53 | import static org.onosproject.openstacknetworking.api.Constants.DIRECT; |
| 54 | import static org.onosproject.openstacknetworking.api.Constants.OPENSTACK_NETWORKING_APP_ID; |
Daniel Park | ec9d113 | 2018-08-19 11:18:03 +0900 | [diff] [blame] | 55 | import static org.onosproject.openstacknetworking.api.Constants.UNSUPPORTED_VENDOR; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 56 | import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.getIntfNameFromPciAddress; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 57 | import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.hasIntfAleadyInDevice; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 58 | import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.COMPUTE; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 59 | import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.CONTROLLER; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 60 | |
| 61 | @Component(immediate = true) |
| 62 | public final class OpenStackSwitchingDirectPortProvider { |
| 63 | private final Logger log = LoggerFactory.getLogger(getClass()); |
| 64 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 65 | private static final String UNBOUND = "unbound"; |
| 66 | private static final String PORT_NAME = "portName"; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 67 | private static final long SLEEP_MS = 3000; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 68 | |
| 69 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 70 | protected CoreService coreService; |
| 71 | |
| 72 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 73 | protected OpenstackNetworkService osNetworkService; |
| 74 | |
| 75 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 76 | protected OpenstackNodeService osNodeService; |
| 77 | |
| 78 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 79 | protected LeadershipService leadershipService; |
| 80 | |
| 81 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 82 | protected ClusterService clusterService; |
| 83 | |
| 84 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 85 | protected DeviceService deviceService; |
| 86 | |
| 87 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 88 | protected InstancePortService instancePortService; |
| 89 | |
| 90 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 91 | protected MastershipService mastershipService; |
| 92 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 93 | private final OpenstackNetworkListener openstackNetworkListener = new InternalOpenstackNetworkListener(); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 94 | private final InternalOpenstackNodeListener internalNodeListener = new InternalOpenstackNodeListener(); |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 95 | |
| 96 | private NodeId localNodeId; |
| 97 | private ApplicationId appId; |
| 98 | |
| 99 | @Activate |
| 100 | void activate() { |
| 101 | appId = coreService.registerApplication(OPENSTACK_NETWORKING_APP_ID); |
| 102 | localNodeId = clusterService.getLocalNode().id(); |
| 103 | leadershipService.runForLeadership(appId.name()); |
| 104 | osNetworkService.addListener(openstackNetworkListener); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 105 | osNodeService.addListener(internalNodeListener); |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 106 | |
| 107 | log.info("Started"); |
| 108 | } |
| 109 | |
| 110 | @Deactivate |
| 111 | void deactivate() { |
| 112 | leadershipService.withdraw(appId.name()); |
| 113 | osNetworkService.removeListener(openstackNetworkListener); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 114 | osNodeService.removeListener(internalNodeListener); |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 115 | |
| 116 | log.info("Stopped"); |
| 117 | } |
| 118 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 119 | private void processPortAdded(Port port) { |
| 120 | if (!port.getvNicType().equals(DIRECT)) { |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 121 | return; |
| 122 | } else if (!port.isAdminStateUp() || port.getVifType().equals(UNBOUND)) { |
| 123 | log.trace("processPortAdded skipped because of status: {}, adminStateUp: {}, vifType: {}", |
| 124 | port.getState(), port.isAdminStateUp(), port.getVifType()); |
| 125 | return; |
| 126 | } else { |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 127 | Optional<OpenstackNode> osNode = osNodeService.completeNodes(COMPUTE).stream() |
| 128 | .filter(node -> node.hostname().equals(port.getHostId())) |
| 129 | .findAny(); |
| 130 | if (!osNode.isPresent()) { |
| 131 | log.error("processPortAdded failed because openstackNode doesn't exist that matches hostname {}", |
| 132 | port.getHostId()); |
| 133 | return; |
| 134 | } |
| 135 | log.trace("Retrieved openstackNode: {}", osNode.get().toString()); |
| 136 | |
| 137 | String intfName = getIntfNameFromPciAddress(port); |
| 138 | if (intfName == null) { |
| 139 | log.error("Failed to execute processPortAdded because of null interface name"); |
| 140 | return; |
Daniel Park | ec9d113 | 2018-08-19 11:18:03 +0900 | [diff] [blame] | 141 | } else if (intfName.equals(UNSUPPORTED_VENDOR)) { |
| 142 | log.warn("Failed to execute processPortAdded because of unsupported vendor for ovs-based sr-iov"); |
| 143 | return; |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 144 | } |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 145 | log.trace("Retrieved interface name: {}", intfName); |
| 146 | |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 147 | try { |
| 148 | //If a VF port has been already added to the device for some reason, we remove it first, |
| 149 | //and the add VF so that other handlers run their logic. |
| 150 | if (hasIntfAleadyInDevice(osNode.get().intgBridge(), |
| 151 | intfName, deviceService)) { |
| 152 | log.trace("Device {} has already has VF interface {}, so remove first.", |
| 153 | osNode.get().intgBridge(), |
| 154 | intfName); |
| 155 | osNodeService.removeVfPort(osNode.get(), intfName); |
| 156 | //we wait 3000ms because the ovsdb client can't deal with removal/add at the same time. |
| 157 | sleep(SLEEP_MS); |
| 158 | } |
| 159 | } catch (InterruptedException e) { |
| 160 | log.error("Exception occurred because of {}", e.toString()); |
| 161 | } |
| 162 | |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 163 | osNodeService.addVfPort(osNode.get(), intfName); |
| 164 | } |
| 165 | } |
| 166 | |
| 167 | private void processPortRemoved(Port port) { |
| 168 | if (!port.getvNicType().equals(DIRECT)) { |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 169 | return; |
| 170 | } else if (instancePortService.instancePort(port.getId()) == null) { |
| 171 | log.trace("processPortRemoved skipped because no instance port exist for portId: {}", port.getId()); |
| 172 | return; |
| 173 | } else { |
| 174 | InstancePort instancePort = instancePortService.instancePort(port.getId()); |
| 175 | if (instancePort == null) { |
| 176 | return; |
| 177 | } |
| 178 | DeviceId deviceId = instancePort.deviceId(); |
| 179 | if (deviceId == null) { |
| 180 | return; |
| 181 | } |
| 182 | OpenstackNode osNode = osNodeService.node(deviceId); |
| 183 | if (osNode == null) { |
| 184 | return; |
| 185 | } |
| 186 | |
| 187 | Optional<org.onosproject.net.Port> removedPort = deviceService.getPorts(deviceId).stream() |
| 188 | .filter(p -> Objects.equals(p.number(), instancePort.portNumber())) |
| 189 | .findAny(); |
| 190 | |
| 191 | if (!removedPort.isPresent()) { |
| 192 | log.error("Failed to execute processPortAdded because port number doesn't exist"); |
| 193 | return; |
| 194 | } |
| 195 | |
| 196 | String intfName = removedPort.get().annotations().value(PORT_NAME); |
| 197 | |
| 198 | if (intfName == null) { |
| 199 | log.error("Failed to execute processPortAdded because of null interface name"); |
| 200 | return; |
| 201 | } |
| 202 | log.trace("Retrieved interface name: {}", intfName); |
| 203 | |
| 204 | osNodeService.removeVfPort(osNode, intfName); |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | private class InternalOpenstackNetworkListener implements OpenstackNetworkListener { |
| 209 | @Override |
| 210 | public boolean isRelevant(OpenstackNetworkEvent event) { |
| 211 | // do not allow to proceed without leadership |
| 212 | NodeId leader = leadershipService.getLeader(appId.name()); |
| 213 | if (!Objects.equals(localNodeId, leader)) { |
| 214 | return false; |
| 215 | } |
| 216 | return true; |
| 217 | } |
| 218 | |
| 219 | @Override |
| 220 | public void event(OpenstackNetworkEvent event) { |
| 221 | switch (event.type()) { |
| 222 | case OPENSTACK_PORT_UPDATED: |
| 223 | if (event.port().getState() == State.DOWN) { |
| 224 | processPortRemoved(event.port()); |
| 225 | } else { |
| 226 | processPortAdded(event.port()); |
| 227 | } |
| 228 | break; |
| 229 | case OPENSTACK_PORT_REMOVED: |
| 230 | processPortRemoved(event.port()); |
| 231 | break; |
| 232 | default: |
| 233 | break; |
| 234 | |
| 235 | } |
| 236 | } |
| 237 | } |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 238 | |
| 239 | private class InternalOpenstackNodeListener implements OpenstackNodeListener { |
| 240 | |
| 241 | @Override |
| 242 | public boolean isRelevant(OpenstackNodeEvent event) { |
| 243 | |
| 244 | if (event.subject().type() == CONTROLLER) { |
| 245 | return false; |
| 246 | } |
| 247 | // do not allow to proceed without mastership |
| 248 | Device device = deviceService.getDevice(event.subject().intgBridge()); |
| 249 | if (device == null) { |
| 250 | return false; |
| 251 | } |
| 252 | return mastershipService.isLocalMaster(device.id()); |
| 253 | } |
| 254 | |
| 255 | @Override |
| 256 | public void event(OpenstackNodeEvent event) { |
| 257 | OpenstackNode osNode = event.subject(); |
| 258 | |
| 259 | switch (event.type()) { |
| 260 | case OPENSTACK_NODE_COMPLETE: |
| 261 | log.info("COMPLETE node {} is detected", osNode.hostname()); |
| 262 | processComputeState(event.subject()); |
| 263 | |
| 264 | break; |
| 265 | case OPENSTACK_NODE_INCOMPLETE: |
| 266 | case OPENSTACK_NODE_CREATED: |
| 267 | case OPENSTACK_NODE_UPDATED: |
| 268 | case OPENSTACK_NODE_REMOVED: |
| 269 | // not reacts to the events other than complete and incomplete states |
| 270 | break; |
| 271 | default: |
| 272 | break; |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | private void processComputeState(OpenstackNode node) { |
| 277 | List<Port> ports = osNetworkService.ports().stream() |
| 278 | .filter(port -> port.getvNicType().equals(DIRECT)) |
| 279 | .filter(port -> !port.getVifType().equals(UNBOUND)) |
| 280 | .filter(port -> port.getHostId().equals(node.hostname())) |
| 281 | .collect(Collectors.toList()); |
| 282 | |
| 283 | ports.forEach(port -> { |
| 284 | addIntfToDevice(node, port); |
| 285 | }); |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 286 | } |
| 287 | |
| 288 | private void addIntfToDevice(OpenstackNode node, Port port) { |
| 289 | String intfName = OpenstackNetworkingUtil.getIntfNameFromPciAddress(port); |
| 290 | if (intfName == null) { |
| 291 | log.error("Failed to retrieve interface name from a port {}", port.getId()); |
Daniel Park | c3e0c60 | 2018-08-30 21:27:25 +0900 | [diff] [blame] | 292 | } else if (intfName.equals(UNSUPPORTED_VENDOR)) { |
| 293 | log.warn("Failed to retrieve interface name from a port {} because of unsupported ovs-based sr-iov"); |
| 294 | return; |
Daniel Park | 95f7331 | 2018-07-31 15:48:34 +0900 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | if (!hasIntfAleadyInDevice(node.intgBridge(), intfName, deviceService)) { |
| 298 | log.debug("Port {} is bound to the interface {} but not added in the bridge {}. Adding it..", |
| 299 | port.getId(), |
| 300 | intfName, |
| 301 | node.intgBridge()); |
| 302 | osNodeService.addVfPort(node, intfName); |
| 303 | } |
| 304 | } |
| 305 | } |
Daniel Park | c4d0640 | 2018-05-28 15:57:37 +0900 | [diff] [blame] | 306 | } |