blob: d4aceeb5a91278549e56a3b0dfc5dde820f08e68 [file] [log] [blame]
Hyunsun Moonb77b60f2016-01-15 20:03:18 -08001/*
2 * Copyright 2014-2015 Open Networking Laboratory
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package org.onosproject.cordvtn;
17
18import com.google.common.collect.Sets;
Hyunsun Moon133fd792016-02-09 01:55:48 -080019import com.jcraft.jsch.Session;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -080020import org.apache.felix.scr.annotations.Activate;
21import org.apache.felix.scr.annotations.Component;
22import org.apache.felix.scr.annotations.Deactivate;
23import org.apache.felix.scr.annotations.Reference;
24import org.apache.felix.scr.annotations.ReferenceCardinality;
25import org.apache.felix.scr.annotations.Service;
Hyunsun Moon133fd792016-02-09 01:55:48 -080026import org.onlab.packet.IpAddress;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -080027import org.onlab.util.ItemNotFoundException;
28import org.onlab.util.KryoNamespace;
29import org.onosproject.cluster.ClusterService;
30import org.onosproject.core.ApplicationId;
31import org.onosproject.core.CoreService;
32import org.onosproject.mastership.MastershipService;
33import org.onosproject.net.ConnectPoint;
34import org.onosproject.net.DefaultAnnotations;
35import org.onosproject.net.Device;
36import org.onosproject.net.DeviceId;
37import org.onosproject.net.Host;
38import org.onosproject.net.Port;
39import org.onosproject.net.behaviour.BridgeConfig;
40import org.onosproject.net.behaviour.BridgeName;
41import org.onosproject.net.behaviour.ControllerInfo;
42import org.onosproject.net.behaviour.DefaultTunnelDescription;
43import org.onosproject.net.behaviour.TunnelConfig;
44import org.onosproject.net.behaviour.TunnelDescription;
45import org.onosproject.net.behaviour.TunnelName;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -080046import org.onosproject.net.config.NetworkConfigEvent;
47import org.onosproject.net.config.NetworkConfigListener;
48import org.onosproject.net.config.NetworkConfigRegistry;
49import org.onosproject.net.config.NetworkConfigService;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -080050import org.onosproject.net.device.DeviceAdminService;
51import org.onosproject.net.device.DeviceEvent;
52import org.onosproject.net.device.DeviceListener;
53import org.onosproject.net.device.DeviceService;
54import org.onosproject.net.driver.DriverHandler;
55import org.onosproject.net.driver.DriverService;
56import org.onosproject.net.flow.FlowRuleService;
57import org.onosproject.net.group.GroupService;
58import org.onosproject.net.host.HostService;
59import org.onosproject.ovsdb.controller.OvsdbClientService;
60import org.onosproject.ovsdb.controller.OvsdbController;
61import org.onosproject.ovsdb.controller.OvsdbNodeId;
62import org.onosproject.store.serializers.KryoNamespaces;
63import org.onosproject.store.service.ConsistentMap;
64import org.onosproject.store.service.Serializer;
65import org.onosproject.store.service.StorageService;
Hyunsun Moon32f3b8e2016-03-02 19:27:26 -080066import org.onosproject.store.service.Versioned;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -080067import org.slf4j.Logger;
68
69import java.util.ArrayList;
70import java.util.HashMap;
71import java.util.List;
72import java.util.Map;
Hyunsun Moon133fd792016-02-09 01:55:48 -080073import java.util.Set;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -080074import java.util.concurrent.ExecutorService;
Hyunsun Moonaf520d32016-03-07 16:37:17 -080075import java.util.stream.Collectors;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -080076
77import static com.google.common.base.Preconditions.checkNotNull;
78import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
79import static org.onlab.util.Tools.groupedThreads;
80import static org.onosproject.net.Device.Type.SWITCH;
81import static org.onosproject.net.behaviour.TunnelDescription.Type.VXLAN;
82import static org.slf4j.LoggerFactory.getLogger;
83
84/**
85 * Reads node information from the network config file and handles the config
86 * update events.
87 * Only a leader controller performs the node addition or deletion.
88 */
89@Component(immediate = true)
90@Service(value = CordVtnNodeManager.class)
91public class CordVtnNodeManager {
92
93 protected final Logger log = getLogger(getClass());
94
95 private static final KryoNamespace.Builder NODE_SERIALIZER = KryoNamespace.newBuilder()
96 .register(KryoNamespaces.API)
Hyunsun Moon133fd792016-02-09 01:55:48 -080097 .register(KryoNamespaces.MISC)
Hyunsun Moonb77b60f2016-01-15 20:03:18 -080098 .register(CordVtnNode.class)
Hyunsun Moon133fd792016-02-09 01:55:48 -080099 .register(NodeState.class)
100 .register(SshAccessInfo.class)
101 .register(NetworkAddress.class);
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800102
103 private static final String DEFAULT_BRIDGE = "br-int";
104 private static final String DEFAULT_TUNNEL = "vxlan";
105 private static final String VPORT_PREFIX = "tap";
106 private static final String OK = "OK";
107 private static final String NO = "NO";
108
109 private static final Map<String, String> DEFAULT_TUNNEL_OPTIONS = new HashMap<String, String>() {
110 {
111 put("key", "flow");
112 put("remote_ip", "flow");
113 }
114 };
115 private static final int DPID_BEGIN = 3;
116 private static final int OFPORT = 6653;
117
118 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
119 protected CoreService coreService;
120
121 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
122 protected NetworkConfigRegistry configRegistry;
123
124 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
125 protected NetworkConfigService configService;
126
127 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
128 protected StorageService storageService;
129
130 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
131 protected DeviceAdminService adminService;
132
133 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
134 protected OvsdbController controller;
135
136 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
137 protected ClusterService clusterService;
138
139 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
140 protected DriverService driverService;
141
142 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
143 protected DeviceService deviceService;
144
145 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
146 protected HostService hostService;
147
148 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
149 protected FlowRuleService flowRuleService;
150
151 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
152 protected MastershipService mastershipService;
153
154 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
155 protected GroupService groupService;
156
157 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
158 protected CordVtnService cordVtnService;
159
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800160 private final ExecutorService eventExecutor =
161 newSingleThreadScheduledExecutor(groupedThreads("onos/cordvtncfg", "event-handler"));
162
163 private final NetworkConfigListener configListener = new InternalConfigListener();
164 private final DeviceListener deviceListener = new InternalDeviceListener();
165
166 private final OvsdbHandler ovsdbHandler = new OvsdbHandler();
167 private final BridgeHandler bridgeHandler = new BridgeHandler();
168
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800169 private ConsistentMap<String, CordVtnNode> nodeStore;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800170 private CordVtnRuleInstaller ruleInstaller;
171 private ApplicationId appId;
172
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800173 private enum NodeState implements CordVtnNodeState {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800174
175 INIT {
176 @Override
177 public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800178 if (!nodeManager.isOvsdbConnected(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800179 nodeManager.connectOvsdb(node);
180 } else {
181 nodeManager.createIntegrationBridge(node);
182 }
183 }
184 },
185 BRIDGE_CREATED {
186 @Override
187 public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800188 if (!nodeManager.isOvsdbConnected(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800189 nodeManager.connectOvsdb(node);
190 } else {
191 nodeManager.createTunnelInterface(node);
Hyunsun Moon133fd792016-02-09 01:55:48 -0800192 nodeManager.addDataPlaneInterface(node);
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800193 }
194 }
195 },
Hyunsun Moon133fd792016-02-09 01:55:48 -0800196 PORTS_ADDED {
Hyunsun Moon177506f2016-01-21 00:54:52 -0800197 @Override
198 public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800199 nodeManager.setIpAddress(node);
Hyunsun Moon177506f2016-01-21 00:54:52 -0800200 }
Hyunsun Moon177506f2016-01-21 00:54:52 -0800201 },
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800202 COMPLETE {
203 @Override
204 public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
205 nodeManager.postInit(node);
206 }
207 },
208 INCOMPLETE {
209 @Override
210 public void process(CordVtnNodeManager nodeManager, CordVtnNode node) {
211 }
212 };
213
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800214 public abstract void process(CordVtnNodeManager nodeManager, CordVtnNode node);
215 }
216
217 @Activate
218 protected void active() {
219 appId = coreService.getAppId(CordVtnService.CORDVTN_APP_ID);
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800220 nodeStore = storageService.<String, CordVtnNode>consistentMapBuilder()
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800221 .withSerializer(Serializer.using(NODE_SERIALIZER.build()))
222 .withName("cordvtn-nodestore")
223 .withApplicationId(appId)
224 .build();
225
226 ruleInstaller = new CordVtnRuleInstaller(appId, flowRuleService,
227 deviceService,
228 driverService,
229 groupService,
230 mastershipService,
231 DEFAULT_TUNNEL);
232
233 deviceService.addListener(deviceListener);
234 configService.addListener(configListener);
Hyunsun Moon746956f2016-01-24 21:47:06 -0800235 readConfiguration();
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800236 }
237
238 @Deactivate
239 protected void deactivate() {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800240 configService.removeListener(configListener);
241 deviceService.removeListener(deviceListener);
242
243 eventExecutor.shutdown();
244 nodeStore.clear();
245 }
246
247 /**
248 * Adds a new node to the service.
249 *
250 * @param node cordvtn node
251 */
252 public void addNode(CordVtnNode node) {
253 checkNotNull(node);
254
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800255 // allow update node attributes
256 nodeStore.put(node.hostname(), CordVtnNode.getUpdatedNode(node, getNodeState(node)));
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800257 initNode(node);
258 }
259
260 /**
261 * Deletes a node from the service.
262 *
263 * @param node cordvtn node
264 */
265 public void deleteNode(CordVtnNode node) {
266 checkNotNull(node);
267
Hyunsun Moon133fd792016-02-09 01:55:48 -0800268 if (isOvsdbConnected(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800269 disconnectOvsdb(node);
270 }
271
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800272 nodeStore.remove(node.hostname());
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800273 }
274
275 /**
276 * Initiates node to serve virtual tenant network.
277 *
278 * @param node cordvtn node
279 */
280 public void initNode(CordVtnNode node) {
281 checkNotNull(node);
282
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800283 if (!nodeStore.containsKey(node.hostname())) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800284 log.warn("Node {} does not exist, add node first", node.hostname());
285 return;
286 }
287
Hyunsun Moon133fd792016-02-09 01:55:48 -0800288 NodeState state = getNodeState(node);
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800289 log.debug("Init node: {} state: {}", node.hostname(), state.toString());
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800290 state.process(this, node);
291 }
292
293 /**
294 * Returns node initialization state.
295 *
296 * @param node cordvtn node
297 * @return true if initial node setup is completed, otherwise false
298 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800299 public boolean isNodeInitComplete(CordVtnNode node) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800300 checkNotNull(node);
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800301 return nodeStore.containsKey(node.hostname()) && getNodeState(node).equals(NodeState.COMPLETE);
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800302 }
303
304 /**
Hyunsun Moon32f3b8e2016-03-02 19:27:26 -0800305 * Returns if current node state saved in nodeStore is COMPLETE or not.
306 *
307 * @param node cordvtn node
308 * @return true if it's complete state, otherwise false
309 */
310 private boolean isNodeStateComplete(CordVtnNode node) {
311 checkNotNull(node);
312
313 // the state saved in nodeStore can be wrong if IP address settings are changed
314 // after the node init has been completed since there's no way to detect it
315 // getNodeState and checkNodeInitState always return correct answer but can be slow
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800316 Versioned<CordVtnNode> versionedNode = nodeStore.get(node.hostname());
317 CordVtnNodeState state = versionedNode.value().state();
318 return state != null && state.equals(NodeState.COMPLETE);
Hyunsun Moon32f3b8e2016-03-02 19:27:26 -0800319 }
320
321 /**
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800322 * Returns detailed node initialization state.
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800323 *
324 * @param node cordvtn node
325 * @return string including detailed node init state
326 */
327 public String checkNodeInitState(CordVtnNode node) {
328 checkNotNull(node);
329
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800330 if (!nodeStore.containsKey(node.hostname())) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800331 log.warn("Node {} does not exist, add node first", node.hostname());
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800332 return null;
333 }
334
Hyunsun Moon133fd792016-02-09 01:55:48 -0800335 Session session = RemoteIpCommandUtil.connect(node.sshInfo());
336 if (session == null) {
337 log.debug("Failed to SSH to {}", node.hostname());
338 return null;
339 }
340
341 Set<IpAddress> intBrIps = RemoteIpCommandUtil.getCurrentIps(session, DEFAULT_BRIDGE);
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800342 String result = String.format(
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800343 "br-int created and connected : %s (%s)%n" +
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800344 "VXLAN interface created : %s%n" +
Hyunsun Moon133fd792016-02-09 01:55:48 -0800345 "Data plane interface added : %s (%s)%n" +
346 "IP flushed from %s : %s%n" +
347 "Data plane IP added to br-int : %s (%s)%n" +
348 "Local management IP added to br-int : %s (%s)",
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800349 isBrIntCreated(node) ? OK : NO, node.intBrId(),
Hyunsun Moon133fd792016-02-09 01:55:48 -0800350 isTunnelIntfCreated(node) ? OK : NO,
351 isDataPlaneIntfAdded(node) ? OK : NO, node.dpIntf(),
352 node.dpIntf(),
353 RemoteIpCommandUtil.getCurrentIps(session, node.dpIntf()).isEmpty() ? OK : NO,
354 intBrIps.contains(node.dpIp().ip()) ? OK : NO, node.dpIp().cidr(),
355 intBrIps.contains(node.localMgmtIp().ip()) ? OK : NO, node.localMgmtIp().cidr());
356
357 RemoteIpCommandUtil.disconnect(session);
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800358
359 return result;
360 }
361
362 /**
363 * Returns the number of the nodes known to the service.
364 *
365 * @return number of nodes
366 */
367 public int getNodeCount() {
368 return nodeStore.size();
369 }
370
371 /**
372 * Returns all nodes known to the service.
373 *
374 * @return list of nodes
375 */
376 public List<CordVtnNode> getNodes() {
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800377 return nodeStore.values().stream()
378 .map(Versioned::value)
379 .collect(Collectors.toList());
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800380 }
381
382 /**
383 * Returns cordvtn node associated with a given OVSDB device.
384 *
385 * @param ovsdbId OVSDB device id
386 * @return cordvtn node, null if it fails to find the node
387 */
388 private CordVtnNode getNodeByOvsdbId(DeviceId ovsdbId) {
389 return getNodes().stream()
390 .filter(node -> node.ovsdbId().equals(ovsdbId))
391 .findFirst().orElse(null);
392 }
393
394 /**
395 * Returns cordvtn node associated with a given integration bridge.
396 *
397 * @param bridgeId device id of integration bridge
398 * @return cordvtn node, null if it fails to find the node
399 */
400 private CordVtnNode getNodeByBridgeId(DeviceId bridgeId) {
401 return getNodes().stream()
402 .filter(node -> node.intBrId().equals(bridgeId))
403 .findFirst().orElse(null);
404 }
405
406 /**
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800407 * Sets a new state for a given cordvtn node.
408 *
409 * @param node cordvtn node
410 * @param newState new node state
411 */
412 private void setNodeState(CordVtnNode node, NodeState newState) {
413 checkNotNull(node);
414
415 log.debug("Changed {} state: {}", node.hostname(), newState.toString());
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800416 nodeStore.put(node.hostname(), CordVtnNode.getUpdatedNode(node, newState));
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800417 newState.process(this, node);
418 }
419
420 /**
421 * Checks current state of a given cordvtn node and returns it.
422 *
423 * @param node cordvtn node
424 * @return node state
425 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800426 private NodeState getNodeState(CordVtnNode node) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800427 checkNotNull(node);
428
Hyunsun Moon133fd792016-02-09 01:55:48 -0800429 if (isBrIntCreated(node) && isTunnelIntfCreated(node) &&
430 isDataPlaneIntfAdded(node) && isIpAddressSet(node)) {
Hyunsun Moon177506f2016-01-21 00:54:52 -0800431 return NodeState.COMPLETE;
Hyunsun Moon133fd792016-02-09 01:55:48 -0800432 } else if (isDataPlaneIntfAdded(node) && isTunnelIntfCreated(node)) {
433 return NodeState.PORTS_ADDED;
434 } else if (isBrIntCreated(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800435 return NodeState.BRIDGE_CREATED;
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800436 } else {
437 return NodeState.INIT;
438 }
439 }
440
441 /**
442 * Performs tasks after node initialization.
443 * It disconnects unnecessary OVSDB connection and installs initial flow
444 * rules on the device.
445 *
446 * @param node cordvtn node
447 */
448 private void postInit(CordVtnNode node) {
449 disconnectOvsdb(node);
450
Hyunsun Moon133fd792016-02-09 01:55:48 -0800451 ruleInstaller.init(node.intBrId(), node.dpIntf(), node.dpIp().ip());
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800452
453 // add existing hosts to the service
454 deviceService.getPorts(node.intBrId()).stream()
455 .filter(port -> getPortName(port).startsWith(VPORT_PREFIX) &&
456 port.isEnabled())
457 .forEach(port -> cordVtnService.addServiceVm(node, getConnectPoint(port)));
458
459 // remove stale hosts from the service
460 hostService.getHosts().forEach(host -> {
461 Port port = deviceService.getPort(host.location().deviceId(), host.location().port());
462 if (port == null) {
463 cordVtnService.removeServiceVm(getConnectPoint(host));
464 }
465 });
466
467 log.info("Finished init {}", node.hostname());
468 }
469
470 /**
471 * Returns port name.
472 *
473 * @param port port
474 * @return port name
475 */
476 private String getPortName(Port port) {
477 return port.annotations().value("portName");
478 }
479
480 /**
481 * Returns connection state of OVSDB server for a given node.
482 *
483 * @param node cordvtn node
484 * @return true if it is connected, false otherwise
485 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800486 private boolean isOvsdbConnected(CordVtnNode node) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800487 checkNotNull(node);
488
489 OvsdbClientService ovsdbClient = getOvsdbClient(node);
490 return deviceService.isAvailable(node.ovsdbId()) &&
491 ovsdbClient != null && ovsdbClient.isConnected();
492 }
493
494 /**
495 * Connects to OVSDB server for a given node.
496 *
497 * @param node cordvtn node
498 */
499 private void connectOvsdb(CordVtnNode node) {
500 checkNotNull(node);
501
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800502 if (!nodeStore.containsKey(node.hostname())) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800503 log.warn("Node {} does not exist", node.hostname());
504 return;
505 }
506
Hyunsun Moon133fd792016-02-09 01:55:48 -0800507 if (!isOvsdbConnected(node)) {
508 controller.connect(node.hostMgmtIp().ip(), node.ovsdbPort());
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800509 }
510 }
511
512 /**
513 * Disconnects OVSDB server for a given node.
514 *
515 * @param node cordvtn node
516 */
517 private void disconnectOvsdb(CordVtnNode node) {
518 checkNotNull(node);
519
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800520 if (!nodeStore.containsKey(node.hostname())) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800521 log.warn("Node {} does not exist", node.hostname());
522 return;
523 }
524
Hyunsun Moon133fd792016-02-09 01:55:48 -0800525 if (isOvsdbConnected(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800526 OvsdbClientService ovsdbClient = getOvsdbClient(node);
527 ovsdbClient.disconnect();
528 }
529 }
530
531 /**
532 * Returns OVSDB client for a given node.
533 *
534 * @param node cordvtn node
535 * @return OVSDB client, or null if it fails to get OVSDB client
536 */
537 private OvsdbClientService getOvsdbClient(CordVtnNode node) {
538 checkNotNull(node);
539
540 OvsdbClientService ovsdbClient = controller.getOvsdbClient(
Hyunsun Moon133fd792016-02-09 01:55:48 -0800541 new OvsdbNodeId(node.hostMgmtIp().ip(), node.ovsdbPort().toInt()));
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800542 if (ovsdbClient == null) {
543 log.trace("Couldn't find OVSDB client for {}", node.hostname());
544 }
545 return ovsdbClient;
546 }
547
548 /**
549 * Creates an integration bridge for a given node.
550 *
551 * @param node cordvtn node
552 */
553 private void createIntegrationBridge(CordVtnNode node) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800554 if (isBrIntCreated(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800555 return;
556 }
557
558 List<ControllerInfo> controllers = new ArrayList<>();
559 Sets.newHashSet(clusterService.getNodes()).stream()
560 .forEach(controller -> {
561 ControllerInfo ctrlInfo = new ControllerInfo(controller.ip(), OFPORT, "tcp");
562 controllers.add(ctrlInfo);
563 });
564
565 String dpid = node.intBrId().toString().substring(DPID_BEGIN);
566
567 try {
568 DriverHandler handler = driverService.createHandler(node.ovsdbId());
569 BridgeConfig bridgeConfig = handler.behaviour(BridgeConfig.class);
570 bridgeConfig.addBridge(BridgeName.bridgeName(DEFAULT_BRIDGE), dpid, controllers);
571 } catch (ItemNotFoundException e) {
Hyunsun Moon177506f2016-01-21 00:54:52 -0800572 log.warn("Failed to create integration bridge on {}", node.hostname());
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800573 }
574 }
575
576 /**
577 * Creates tunnel interface to the integration bridge for a given node.
578 *
579 * @param node cordvtn node
580 */
581 private void createTunnelInterface(CordVtnNode node) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800582 if (isTunnelIntfCreated(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800583 return;
584 }
585
586 DefaultAnnotations.Builder optionBuilder = DefaultAnnotations.builder();
587 for (String key : DEFAULT_TUNNEL_OPTIONS.keySet()) {
588 optionBuilder.set(key, DEFAULT_TUNNEL_OPTIONS.get(key));
589 }
590
591 TunnelDescription description = new DefaultTunnelDescription(
592 null, null, VXLAN, TunnelName.tunnelName(DEFAULT_TUNNEL),
593 optionBuilder.build());
594
595 try {
596 DriverHandler handler = driverService.createHandler(node.ovsdbId());
597 TunnelConfig tunnelConfig = handler.behaviour(TunnelConfig.class);
598 tunnelConfig.createTunnelInterface(BridgeName.bridgeName(DEFAULT_BRIDGE), description);
599 } catch (ItemNotFoundException e) {
Hyunsun Moon177506f2016-01-21 00:54:52 -0800600 log.warn("Failed to create tunnel interface on {}", node.hostname());
601 }
602 }
603
604 /**
Hyunsun Moon133fd792016-02-09 01:55:48 -0800605 * Adds data plane interface to a given node.
Hyunsun Moon177506f2016-01-21 00:54:52 -0800606 *
607 * @param node cordvtn node
608 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800609 private void addDataPlaneInterface(CordVtnNode node) {
610 if (isDataPlaneIntfAdded(node)) {
Hyunsun Moon177506f2016-01-21 00:54:52 -0800611 return;
612 }
613
614 try {
615 DriverHandler handler = driverService.createHandler(node.ovsdbId());
616 BridgeConfig bridgeConfig = handler.behaviour(BridgeConfig.class);
Hyunsun Moon133fd792016-02-09 01:55:48 -0800617 bridgeConfig.addPort(BridgeName.bridgeName(DEFAULT_BRIDGE), node.dpIntf());
Hyunsun Moon177506f2016-01-21 00:54:52 -0800618 } catch (ItemNotFoundException e) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800619 log.warn("Failed to add {} on {}", node.dpIntf(), node.hostname());
620 }
621 }
622
623 /**
624 * Flushes IP address from data plane interface and adds data plane IP address
625 * to integration bridge.
626 *
627 * @param node cordvtn node
628 */
629 private void setIpAddress(CordVtnNode node) {
630 Session session = RemoteIpCommandUtil.connect(node.sshInfo());
631 if (session == null) {
632 log.debug("Failed to SSH to {}", node.hostname());
633 return;
634 }
635
Hyunsun Moone9d4f4a2016-03-04 19:24:08 -0800636 RemoteIpCommandUtil.getCurrentIps(session, DEFAULT_BRIDGE).stream()
637 .filter(ip -> !ip.equals(node.localMgmtIp().ip()))
638 .filter(ip -> !ip.equals(node.dpIp().ip()))
639 .forEach(ip -> RemoteIpCommandUtil.deleteIp(session, ip, DEFAULT_BRIDGE));
640
Hyunsun Moon133fd792016-02-09 01:55:48 -0800641 boolean result = RemoteIpCommandUtil.flushIp(session, node.dpIntf()) &&
642 RemoteIpCommandUtil.setInterfaceUp(session, node.dpIntf()) &&
643 RemoteIpCommandUtil.addIp(session, node.dpIp(), DEFAULT_BRIDGE) &&
644 RemoteIpCommandUtil.addIp(session, node.localMgmtIp(), DEFAULT_BRIDGE) &&
645 RemoteIpCommandUtil.setInterfaceUp(session, DEFAULT_BRIDGE);
646
647 RemoteIpCommandUtil.disconnect(session);
648
649 if (result) {
650 setNodeState(node, NodeState.COMPLETE);
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800651 }
652 }
653
654 /**
655 * Checks if integration bridge exists and available.
656 *
657 * @param node cordvtn node
658 * @return true if the bridge is available, false otherwise
659 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800660 private boolean isBrIntCreated(CordVtnNode node) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800661 return (deviceService.getDevice(node.intBrId()) != null
662 && deviceService.isAvailable(node.intBrId()));
663 }
664
665 /**
666 * Checks if tunnel interface exists.
667 *
668 * @param node cordvtn node
669 * @return true if the interface exists, false otherwise
670 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800671 private boolean isTunnelIntfCreated(CordVtnNode node) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800672 return deviceService.getPorts(node.intBrId())
673 .stream()
674 .filter(p -> getPortName(p).contains(DEFAULT_TUNNEL) &&
675 p.isEnabled())
676 .findAny().isPresent();
677 }
678
679 /**
Hyunsun Moon133fd792016-02-09 01:55:48 -0800680 * Checks if data plane interface exists.
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800681 *
682 * @param node cordvtn node
683 * @return true if the interface exists, false otherwise
684 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800685 private boolean isDataPlaneIntfAdded(CordVtnNode node) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800686 return deviceService.getPorts(node.intBrId())
687 .stream()
Hyunsun Moon133fd792016-02-09 01:55:48 -0800688 .filter(p -> getPortName(p).contains(node.dpIntf()) &&
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800689 p.isEnabled())
690 .findAny().isPresent();
691 }
692
693 /**
Hyunsun Moon133fd792016-02-09 01:55:48 -0800694 * Checks if the IP addresses are correctly set.
695 *
696 * @param node cordvtn node
697 * @return true if the IP is set, false otherwise
698 */
699 private boolean isIpAddressSet(CordVtnNode node) {
700 Session session = RemoteIpCommandUtil.connect(node.sshInfo());
701 if (session == null) {
702 log.debug("Failed to SSH to {}", node.hostname());
703 return false;
704 }
705
706 Set<IpAddress> intBrIps = RemoteIpCommandUtil.getCurrentIps(session, DEFAULT_BRIDGE);
707 boolean result = RemoteIpCommandUtil.getCurrentIps(session, node.dpIntf()).isEmpty() &&
708 RemoteIpCommandUtil.isInterfaceUp(session, node.dpIntf()) &&
709 intBrIps.contains(node.dpIp().ip()) &&
710 intBrIps.contains(node.localMgmtIp().ip()) &&
711 RemoteIpCommandUtil.isInterfaceUp(session, DEFAULT_BRIDGE);
712
713 RemoteIpCommandUtil.disconnect(session);
714 return result;
715 }
716
717 /**
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800718 * Returns connect point of a given port.
719 *
720 * @param port port
721 * @return connect point
722 */
723 private ConnectPoint getConnectPoint(Port port) {
724 return new ConnectPoint(port.element().id(), port.number());
725 }
726
727 /**
728 * Returns connect point of a given host.
729 *
730 * @param host host
731 * @return connect point
732 */
733 private ConnectPoint getConnectPoint(Host host) {
734 return new ConnectPoint(host.location().deviceId(), host.location().port());
735 }
736
737 private class OvsdbHandler implements ConnectionHandler<Device> {
738
739 @Override
740 public void connected(Device device) {
741 CordVtnNode node = getNodeByOvsdbId(device.id());
742 if (node != null) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800743 setNodeState(node, getNodeState(node));
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800744 } else {
745 log.debug("{} is detected on unregistered node, ignore it.", device.id());
746 }
747 }
748
749 @Override
750 public void disconnected(Device device) {
751 if (!deviceService.isAvailable(device.id())) {
Hyunsun Moonaf520d32016-03-07 16:37:17 -0800752 log.debug("Device {} is disconnected", device.id());
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800753 adminService.removeDevice(device.id());
754 }
755 }
756 }
757
758 private class BridgeHandler implements ConnectionHandler<Device> {
759
760 @Override
761 public void connected(Device device) {
762 CordVtnNode node = getNodeByBridgeId(device.id());
763 if (node != null) {
Hyunsun Moon133fd792016-02-09 01:55:48 -0800764 setNodeState(node, getNodeState(node));
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800765 } else {
766 log.debug("{} is detected on unregistered node, ignore it.", device.id());
767 }
768 }
769
770 @Override
771 public void disconnected(Device device) {
772 CordVtnNode node = getNodeByBridgeId(device.id());
773 if (node != null) {
774 log.debug("Integration Bridge is disconnected from {}", node.hostname());
775 setNodeState(node, NodeState.INCOMPLETE);
776 }
777 }
778
779 /**
780 * Handles port added situation.
Hyunsun Moon133fd792016-02-09 01:55:48 -0800781 * If the added port is tunnel or data plane interface, proceed to the remaining
782 * node initialization. Otherwise, do nothing.
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800783 *
784 * @param port port
785 */
786 public void portAdded(Port port) {
787 CordVtnNode node = getNodeByBridgeId((DeviceId) port.element().id());
788 String portName = getPortName(port);
789
790 if (node == null) {
791 log.debug("{} is added to unregistered node, ignore it.", portName);
792 return;
793 }
794
795 log.debug("Port {} is added to {}", portName, node.hostname());
796
797 if (portName.startsWith(VPORT_PREFIX)) {
Hyunsun Moon32f3b8e2016-03-02 19:27:26 -0800798 if (isNodeStateComplete(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800799 cordVtnService.addServiceVm(node, getConnectPoint(port));
800 } else {
801 log.debug("VM is detected on incomplete node, ignore it.", portName);
802 }
Hyunsun Moon133fd792016-02-09 01:55:48 -0800803 } else if (portName.contains(DEFAULT_TUNNEL) || portName.equals(node.dpIntf())) {
804 setNodeState(node, getNodeState(node));
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800805 }
806 }
807
808 /**
809 * Handles port removed situation.
Hyunsun Moon133fd792016-02-09 01:55:48 -0800810 * If the removed port is tunnel or data plane interface, proceed to the remaining
811 * node initialization.Others, do nothing.
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800812 *
813 * @param port port
814 */
815 public void portRemoved(Port port) {
816 CordVtnNode node = getNodeByBridgeId((DeviceId) port.element().id());
817 String portName = getPortName(port);
818
819 if (node == null) {
820 return;
821 }
822
823 log.debug("Port {} is removed from {}", portName, node.hostname());
824
825 if (portName.startsWith(VPORT_PREFIX)) {
Hyunsun Moon32f3b8e2016-03-02 19:27:26 -0800826 if (isNodeStateComplete(node)) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800827 cordVtnService.removeServiceVm(getConnectPoint(port));
828 } else {
829 log.debug("VM is vanished from incomplete node, ignore it.", portName);
830 }
Hyunsun Moon133fd792016-02-09 01:55:48 -0800831 } else if (portName.contains(DEFAULT_TUNNEL) || portName.equals(node.dpIntf())) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800832 setNodeState(node, NodeState.INCOMPLETE);
833 }
834 }
835 }
836
837 private class InternalDeviceListener implements DeviceListener {
838
839 @Override
840 public void event(DeviceEvent event) {
841
842 Device device = event.subject();
843 ConnectionHandler<Device> handler =
844 (device.type().equals(SWITCH) ? bridgeHandler : ovsdbHandler);
845
846 switch (event.type()) {
847 case PORT_ADDED:
848 eventExecutor.submit(() -> bridgeHandler.portAdded(event.port()));
849 break;
850 case PORT_UPDATED:
851 if (!event.port().isEnabled()) {
852 eventExecutor.submit(() -> bridgeHandler.portRemoved(event.port()));
853 }
854 break;
855 case DEVICE_ADDED:
856 case DEVICE_AVAILABILITY_CHANGED:
857 if (deviceService.isAvailable(device.id())) {
858 eventExecutor.submit(() -> handler.connected(device));
859 } else {
860 eventExecutor.submit(() -> handler.disconnected(device));
861 }
862 break;
863 default:
864 break;
865 }
866 }
867 }
868
869 /**
Hyunsun Moon746956f2016-01-24 21:47:06 -0800870 * Reads cordvtn nodes from config file.
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800871 */
872 private void readConfiguration() {
873 CordVtnConfig config = configRegistry.getConfig(appId, CordVtnConfig.class);
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800874 if (config == null) {
Hyunsun Moon746956f2016-01-24 21:47:06 -0800875 log.debug("No configuration found");
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800876 return;
877 }
878
Hyunsun Moon32f3b8e2016-03-02 19:27:26 -0800879 config.cordVtnNodes().forEach(this::addNode);
Hyunsun Moon746956f2016-01-24 21:47:06 -0800880 // TODO remove nodes if needed
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800881 }
882
883 private class InternalConfigListener implements NetworkConfigListener {
884
885 @Override
886 public void event(NetworkConfigEvent event) {
887 if (!event.configClass().equals(CordVtnConfig.class)) {
888 return;
889 }
890
891 switch (event.type()) {
892 case CONFIG_ADDED:
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800893 case CONFIG_UPDATED:
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800894 eventExecutor.execute(CordVtnNodeManager.this::readConfiguration);
895 break;
896 default:
897 break;
898 }
899 }
900 }
901}