blob: 93e76adb038c622d79e3bee84888eef4eb93fd40 [file] [log] [blame]
Thomas Vachuska8fd25052015-09-10 16:15:33 -07001/*
Brian O'Connor0947d7e2017-08-03 21:12:30 -07002 * Copyright 2015-present Open Networking Foundation
Thomas Vachuska8fd25052015-09-10 16:15:33 -07003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
Charles Chan319d1a22015-11-03 10:42:14 -080016package org.onosproject.segmentrouting.config;
sangho80f11cb2015-04-01 13:05:26 -070017
Charles Chan6191aca2017-09-12 12:09:22 -070018import com.fasterxml.jackson.databind.JsonNode;
19import com.fasterxml.jackson.databind.node.ArrayNode;
20import com.fasterxml.jackson.databind.node.ObjectNode;
Charles Chan82ab1932016-01-30 23:22:37 -080021import com.google.common.collect.HashMultimap;
Charles Chanc6ad7752015-10-29 14:58:10 -070022import com.google.common.collect.ImmutableSet;
Charles Chandc7d6f92018-03-20 13:30:38 -070023import com.google.common.collect.Multimaps;
Charles Chan82ab1932016-01-30 23:22:37 -080024import com.google.common.collect.SetMultimap;
sangho80f11cb2015-04-01 13:05:26 -070025import org.onlab.packet.Ip4Address;
Pier Ventreadb4ae62016-11-23 09:57:42 -080026import org.onlab.packet.Ip6Address;
Charles Chandebfea32016-10-24 14:52:01 -070027import org.onlab.packet.IpAddress;
Charles Chan82ab1932016-01-30 23:22:37 -080028import org.onlab.packet.IpPrefix;
sangho80f11cb2015-04-01 13:05:26 -070029import org.onlab.packet.MacAddress;
Charles Chanb7f75ac2016-01-11 18:28:54 -080030import org.onlab.packet.VlanId;
Charles Chane7c61022015-10-07 14:21:45 -070031import org.onosproject.net.ConnectPoint;
sangho80f11cb2015-04-01 13:05:26 -070032import org.onosproject.net.DeviceId;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070033import org.onosproject.net.PortNumber;
Jonathan Hart61e24e12017-11-30 18:23:42 -080034import org.onosproject.net.config.ConfigException;
35import org.onosproject.net.config.basics.InterfaceConfig;
36import org.onosproject.net.host.InterfaceIpAddress;
37import org.onosproject.net.intf.Interface;
Charles Chan46fdfaf2016-11-09 20:51:44 -080038import org.onosproject.segmentrouting.SegmentRoutingManager;
sangho80f11cb2015-04-01 13:05:26 -070039import org.slf4j.Logger;
40import org.slf4j.LoggerFactory;
41
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070042import java.util.ArrayList;
Jonathan Hart61e24e12017-11-30 18:23:42 -080043import java.util.Collection;
Pier Ventreb6a7f342016-11-26 21:05:22 -080044import java.util.Collections;
sangho80f11cb2015-04-01 13:05:26 -070045import java.util.HashMap;
Charles Chan9bec8a32015-12-01 10:00:51 -080046import java.util.HashSet;
sangho80f11cb2015-04-01 13:05:26 -070047import java.util.List;
48import java.util.Map;
Charles Chancf789822019-03-22 10:04:27 -070049import java.util.Objects;
Charles Chan72f556a2015-10-05 17:50:33 -070050import java.util.Set;
Saurav Das7c305372015-10-28 12:39:42 -070051import java.util.concurrent.ConcurrentHashMap;
Charles Chan46fdfaf2016-11-09 20:51:44 -080052import java.util.stream.Collectors;
sangho80f11cb2015-04-01 13:05:26 -070053
Charles Chanc22cef32016-04-29 14:38:22 -070054import static com.google.common.base.Preconditions.checkNotNull;
55
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070056/**
57 * Segment Routing configuration component that reads the
58 * segment routing related configuration from Network Configuration Manager
59 * component and organizes in more accessible formats.
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070060 */
sangho80f11cb2015-04-01 13:05:26 -070061public class DeviceConfiguration implements DeviceProperties {
62
Charles Chan46fdfaf2016-11-09 20:51:44 -080063 private static final String NO_SUBNET = "No subnet configured on {}";
64
Charles Chan43547ca2016-02-10 20:46:58 -080065 private static final Logger log = LoggerFactory.getLogger(DeviceConfiguration.class);
Sho SHIMIZU47b8aa22015-09-11 11:19:11 -070066 private final List<Integer> allSegmentIds = new ArrayList<>();
Charles Chanb7f75ac2016-01-11 18:28:54 -080067 private final Map<DeviceId, SegmentRouterInfo> deviceConfigMap = new ConcurrentHashMap<>();
Charles Chan46fdfaf2016-11-09 20:51:44 -080068 private SegmentRoutingManager srManager;
sangho80f11cb2015-04-01 13:05:26 -070069
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070070 private class SegmentRouterInfo {
Charles Chan3d35a542018-10-02 12:52:29 -070071 int ipv4NodeSid = -1;
72 int ipv6NodeSid = -1;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070073 DeviceId deviceId;
Pier Ventreadb4ae62016-11-23 09:57:42 -080074 Ip4Address ipv4Loopback;
75 Ip6Address ipv6Loopback;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070076 MacAddress mac;
77 boolean isEdge;
Pier Ventreb6a7f342016-11-26 21:05:22 -080078 SetMultimap<PortNumber, IpAddress> gatewayIps;
79 SetMultimap<PortNumber, IpPrefix> subnets;
Charles Chan9bec8a32015-12-01 10:00:51 -080080 Map<Integer, Set<Integer>> adjacencySids;
Saurav Das261c3002017-06-13 15:35:54 -070081 DeviceId pairDeviceId;
82 PortNumber pairLocalPort;
Andreas Pantelopoulos5e7be3d2017-10-23 12:18:25 -070083 int pwRoutingLabel;
Charles Chan2b078ae2015-10-14 11:24:40 -070084
85 public SegmentRouterInfo() {
Charles Chandc7d6f92018-03-20 13:30:38 -070086 gatewayIps = Multimaps.synchronizedSetMultimap(HashMultimap.create());
87 subnets = Multimaps.synchronizedSetMultimap(HashMultimap.create());
Charles Chan2b078ae2015-10-14 11:24:40 -070088 }
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070089 }
sangho80f11cb2015-04-01 13:05:26 -070090
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070091 /**
Charles Chanb7f75ac2016-01-11 18:28:54 -080092 * Constructs device configuration for all Segment Router devices,
93 * organizing the data into various maps for easier access.
Brian O'Connor65eeb572015-10-09 17:03:44 -070094 *
Charles Chan46fdfaf2016-11-09 20:51:44 -080095 * @param srManager Segment Routing Manager
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -070096 */
Charles Chan46fdfaf2016-11-09 20:51:44 -080097 public DeviceConfiguration(SegmentRoutingManager srManager) {
98 this.srManager = srManager;
Saurav Das261c3002017-06-13 15:35:54 -070099 updateConfig();
100 }
Charles Chan57bd98c2016-02-22 19:27:29 -0800101
Saurav Das261c3002017-06-13 15:35:54 -0700102 public void updateConfig() {
Charles Chane7c61022015-10-07 14:21:45 -0700103 // Read config from device subject, excluding gatewayIps and subnets.
104 Set<DeviceId> deviceSubjects =
Charles Chan46fdfaf2016-11-09 20:51:44 -0800105 srManager.cfgService.getSubjects(DeviceId.class, SegmentRoutingDeviceConfig.class);
Charles Chane7c61022015-10-07 14:21:45 -0700106 deviceSubjects.forEach(subject -> {
Charles Chan82ab1932016-01-30 23:22:37 -0800107 SegmentRoutingDeviceConfig config =
Charles Chan46fdfaf2016-11-09 20:51:44 -0800108 srManager.cfgService.getConfig(subject, SegmentRoutingDeviceConfig.class);
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700109 SegmentRouterInfo info = new SegmentRouterInfo();
Charles Chan72f556a2015-10-05 17:50:33 -0700110 info.deviceId = subject;
Pier Ventreadb4ae62016-11-23 09:57:42 -0800111 info.ipv4NodeSid = config.nodeSidIPv4();
112 info.ipv6NodeSid = config.nodeSidIPv6();
113 info.ipv4Loopback = config.routerIpv4();
114 info.ipv6Loopback = config.routerIpv6();
Charles Chan9bec8a32015-12-01 10:00:51 -0800115 info.mac = config.routerMac();
Charles Chan72f556a2015-10-05 17:50:33 -0700116 info.isEdge = config.isEdgeRouter();
Charles Chan9bec8a32015-12-01 10:00:51 -0800117 info.adjacencySids = config.adjacencySids();
Saurav Das261c3002017-06-13 15:35:54 -0700118 info.pairDeviceId = config.pairDeviceId();
119 info.pairLocalPort = config.pairLocalPort();
Andreas Pantelopoulos5e7be3d2017-10-23 12:18:25 -0700120 info.pwRoutingLabel = info.ipv4NodeSid + 1000;
Charles Chanb7f75ac2016-01-11 18:28:54 -0800121 deviceConfigMap.put(info.deviceId, info);
Charles Chan68363b12017-06-26 15:25:09 -0700122 log.debug("Read device config for device: {}", info.deviceId);
Pier Ventreadb4ae62016-11-23 09:57:42 -0800123 /*
124 * IPv6 sid is not inserted. this part of the code is not used for now.
125 */
126 allSegmentIds.add(info.ipv4NodeSid);
Charles Chan72f556a2015-10-05 17:50:33 -0700127 });
Charles Chane7c61022015-10-07 14:21:45 -0700128
Charles Chan46fdfaf2016-11-09 20:51:44 -0800129 // Read gatewayIps and subnets from port subject. Ignore suppressed ports.
130 Set<ConnectPoint> portSubjects = srManager.cfgService
131 .getSubjects(ConnectPoint.class, InterfaceConfig.class);
Jonathan Hart61e24e12017-11-30 18:23:42 -0800132 portSubjects.stream()
133 .filter(subject -> deviceConfigMap.containsKey(subject.deviceId()))
134 .filter(subject -> !isSuppressedPort(subject)).forEach(subject -> {
Charles Chane7c61022015-10-07 14:21:45 -0700135 InterfaceConfig config =
Charles Chan46fdfaf2016-11-09 20:51:44 -0800136 srManager.cfgService.getConfig(subject, InterfaceConfig.class);
Charles Chane7c61022015-10-07 14:21:45 -0700137 Set<Interface> networkInterfaces;
138 try {
139 networkInterfaces = config.getInterfaces();
140 } catch (ConfigException e) {
141 log.error("Error loading port configuration");
142 return;
143 }
144 networkInterfaces.forEach(networkInterface -> {
Charles Chanb7f75ac2016-01-11 18:28:54 -0800145 VlanId vlanId = networkInterface.vlan();
146 ConnectPoint connectPoint = networkInterface.connectPoint();
147 DeviceId dpid = connectPoint.deviceId();
148 PortNumber port = connectPoint.port();
Charles Chan6191aca2017-09-12 12:09:22 -0700149 MacAddress mac = networkInterface.mac();
Charles Chanb7f75ac2016-01-11 18:28:54 -0800150 SegmentRouterInfo info = deviceConfigMap.get(dpid);
Charles Chane7c61022015-10-07 14:21:45 -0700151
Charles Chan2b078ae2015-10-14 11:24:40 -0700152 // skip if there is no corresponding device for this ConenctPoint
153 if (info != null) {
Charles Chanb7f75ac2016-01-11 18:28:54 -0800154 // Extract subnet information
Jonathan Hart8fa9ec52016-02-16 10:30:37 -0800155 List<InterfaceIpAddress> interfaceAddresses = networkInterface.ipAddressesList();
Charles Chan2b078ae2015-10-14 11:24:40 -0700156 interfaceAddresses.forEach(interfaceAddress -> {
Pier Ventreb6a7f342016-11-26 21:05:22 -0800157 // Do not add /0, /32 and /128 to gateway IP list
Charles Chan82ab1932016-01-30 23:22:37 -0800158 int prefixLength = interfaceAddress.subnetAddress().prefixLength();
Pier Ventreb6a7f342016-11-26 21:05:22 -0800159 IpPrefix ipPrefix = interfaceAddress.subnetAddress();
160 if (ipPrefix.isIp4()) {
161 if (prefixLength != 0 && prefixLength != IpPrefix.MAX_INET_MASK_LENGTH) {
162 info.gatewayIps.put(port, interfaceAddress.ipAddress());
163 }
164 info.subnets.put(port, interfaceAddress.subnetAddress());
165 } else {
166 if (prefixLength != 0 && prefixLength != IpPrefix.MAX_INET6_MASK_LENGTH) {
167 info.gatewayIps.put(port, interfaceAddress.ipAddress());
168 }
169 info.subnets.put(port, interfaceAddress.subnetAddress());
Charles Chan82ab1932016-01-30 23:22:37 -0800170 }
Charles Chan2b078ae2015-10-14 11:24:40 -0700171 });
Charles Chan6191aca2017-09-12 12:09:22 -0700172
173 // Override interface mac with router mac
174 if (!mac.equals(info.mac)) {
175 ArrayNode array = (ArrayNode) config.node();
176 for (JsonNode intfNode : array) {
177 ObjectNode objNode = (ObjectNode) intfNode;
178 objNode.put(InterfaceConfig.MAC, info.mac.toString());
179 }
180 srManager.cfgService.applyConfig(connectPoint, InterfaceConfig.class, array);
181 }
Charles Chan2b078ae2015-10-14 11:24:40 -0700182 }
Charles Chane7c61022015-10-07 14:21:45 -0700183 });
Pier Luigi6a2643a2017-01-31 09:35:05 -0800184 // We register the connect point with the NRS.
Pier Ventreb6a7f342016-11-26 21:05:22 -0800185 srManager.registerConnectPoint(subject);
Charles Chane7c61022015-10-07 14:21:45 -0700186 });
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700187 }
188
Jonathan Hart61e24e12017-11-30 18:23:42 -0800189 public Collection<DeviceId> getRouters() {
190 return deviceConfigMap.keySet();
191 }
Saurav Das261c3002017-06-13 15:35:54 -0700192
sangho80f11cb2015-04-01 13:05:26 -0700193 @Override
Charles Chan319d1a22015-11-03 10:42:14 -0800194 public boolean isConfigured(DeviceId deviceId) {
195 return deviceConfigMap.get(deviceId) != null;
196 }
197
198 @Override
Pier Ventreadb4ae62016-11-23 09:57:42 -0800199 public int getIPv4SegmentId(DeviceId deviceId) throws DeviceConfigNotFoundException {
Saurav Das7c305372015-10-28 12:39:42 -0700200 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
201 if (srinfo != null) {
Pier Ventreadb4ae62016-11-23 09:57:42 -0800202 log.trace("getIPv4SegmentId for device{} is {}", deviceId, srinfo.ipv4NodeSid);
203 return srinfo.ipv4NodeSid;
sangho80f11cb2015-04-01 13:05:26 -0700204 } else {
Pier Ventreadb4ae62016-11-23 09:57:42 -0800205 String message = "getIPv4SegmentId fails for device: " + deviceId + ".";
206 throw new DeviceConfigNotFoundException(message);
207 }
208 }
209
210 @Override
211 public int getIPv6SegmentId(DeviceId deviceId) throws DeviceConfigNotFoundException {
212 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
213 if (srinfo != null) {
214 log.trace("getIPv6SegmentId for device{} is {}", deviceId, srinfo.ipv6NodeSid);
215 return srinfo.ipv6NodeSid;
216 } else {
217 String message = "getIPv6SegmentId fails for device: " + deviceId + ".";
Charles Chan319d1a22015-11-03 10:42:14 -0800218 throw new DeviceConfigNotFoundException(message);
sangho80f11cb2015-04-01 13:05:26 -0700219 }
220 }
221
Andreas Pantelopoulos5e7be3d2017-10-23 12:18:25 -0700222 @Override
223 public int getPWRoutingLabel(DeviceId deviceId) throws DeviceConfigNotFoundException {
224 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
225 if (srinfo != null) {
226 log.trace("pwRoutingLabel for device{} is {}", deviceId, srinfo.pwRoutingLabel);
227 return srinfo.pwRoutingLabel;
228 } else {
229 String message = "getPWRoutingLabel fails for device: " + deviceId + ".";
230 throw new DeviceConfigNotFoundException(message);
231 }
232 }
233
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700234 /**
Pier Ventreadb4ae62016-11-23 09:57:42 -0800235 * Returns the IPv4 Node segment id of a segment router given its Router mac address.
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700236 *
237 * @param routerMac router mac address
Saurav Das7c305372015-10-28 12:39:42 -0700238 * @return node segment id, or -1 if not found in config
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700239 */
Pier Ventreadb4ae62016-11-23 09:57:42 -0800240 public int getIPv4SegmentId(MacAddress routerMac) {
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700241 for (Map.Entry<DeviceId, SegmentRouterInfo> entry:
242 deviceConfigMap.entrySet()) {
243 if (entry.getValue().mac.equals(routerMac)) {
Pier Ventreadb4ae62016-11-23 09:57:42 -0800244 return entry.getValue().ipv4NodeSid;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700245 }
246 }
sangho80f11cb2015-04-01 13:05:26 -0700247
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700248 return -1;
249 }
250
251 /**
Pier Ventreadb4ae62016-11-23 09:57:42 -0800252 * Returns the IPv6 Node segment id of a segment router given its Router mac address.
253 *
254 * @param routerMac router mac address
255 * @return node segment id, or -1 if not found in config
256 */
257 public int getIPv6SegmentId(MacAddress routerMac) {
258 for (Map.Entry<DeviceId, SegmentRouterInfo> entry:
259 deviceConfigMap.entrySet()) {
260 if (entry.getValue().mac.equals(routerMac)) {
261 return entry.getValue().ipv6NodeSid;
262 }
263 }
264
265 return -1;
266 }
267
268 /**
269 * Returns the IPv4 Node segment id of a segment router given its Router ip address.
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700270 *
271 * @param routerAddress router ip address
Saurav Das7c305372015-10-28 12:39:42 -0700272 * @return node segment id, or -1 if not found in config
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700273 */
Pier Ventreadb4ae62016-11-23 09:57:42 -0800274 public int getIPv4SegmentId(Ip4Address routerAddress) {
Charles Chan3d35a542018-10-02 12:52:29 -0700275 for (Map.Entry<DeviceId, SegmentRouterInfo> entry: deviceConfigMap.entrySet()) {
276 Ip4Address ipv4Loopback = entry.getValue().ipv4Loopback;
277 if (ipv4Loopback == null) {
278 continue;
279 }
Pier Ventreadb4ae62016-11-23 09:57:42 -0800280 if (entry.getValue().ipv4Loopback.equals(routerAddress)) {
Charles Chan3d35a542018-10-02 12:52:29 -0700281 if (entry.getValue().ipv4NodeSid == -1) {
282 continue;
283 }
Pier Ventreadb4ae62016-11-23 09:57:42 -0800284 return entry.getValue().ipv4NodeSid;
285 }
286 }
287
288 return -1;
289 }
290
291 /**
292 * Returns the IPv6 Node segment id of a segment router given its Router ip address.
293 *
294 * @param routerAddress router ip address
295 * @return node segment id, or -1 if not found in config
296 */
297 public int getIPv6SegmentId(Ip6Address routerAddress) {
Charles Chan3d35a542018-10-02 12:52:29 -0700298 for (Map.Entry<DeviceId, SegmentRouterInfo> entry: deviceConfigMap.entrySet()) {
299 Ip6Address ipv6Loopback = entry.getValue().ipv6Loopback;
300 if (ipv6Loopback == null) {
301 continue;
302 }
Pier Ventreadb4ae62016-11-23 09:57:42 -0800303 if (entry.getValue().ipv6Loopback.equals(routerAddress)) {
Charles Chan3d35a542018-10-02 12:52:29 -0700304 if (entry.getValue().ipv6NodeSid == -1) {
305 continue;
306 }
Pier Ventreadb4ae62016-11-23 09:57:42 -0800307 return entry.getValue().ipv6NodeSid;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700308 }
309 }
310
311 return -1;
312 }
313
sangho80f11cb2015-04-01 13:05:26 -0700314 @Override
Charles Chan319d1a22015-11-03 10:42:14 -0800315 public MacAddress getDeviceMac(DeviceId deviceId) throws DeviceConfigNotFoundException {
Saurav Das7c305372015-10-28 12:39:42 -0700316 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
317 if (srinfo != null) {
Saurav Das7c305372015-10-28 12:39:42 -0700318 return srinfo.mac;
sangho80f11cb2015-04-01 13:05:26 -0700319 } else {
Charles Chan319d1a22015-11-03 10:42:14 -0800320 String message = "getDeviceMac fails for device: " + deviceId + ".";
321 throw new DeviceConfigNotFoundException(message);
sangho80f11cb2015-04-01 13:05:26 -0700322 }
323 }
324
Charles Chan319d1a22015-11-03 10:42:14 -0800325 @Override
Pier Ventreadb4ae62016-11-23 09:57:42 -0800326 public Ip4Address getRouterIpv4(DeviceId deviceId) throws DeviceConfigNotFoundException {
Saurav Das7c305372015-10-28 12:39:42 -0700327 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
328 if (srinfo != null) {
Pier Ventreadb4ae62016-11-23 09:57:42 -0800329 log.trace("getRouterIpv4 for device{} is {}", deviceId, srinfo.ipv4Loopback);
330 return srinfo.ipv4Loopback;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700331 } else {
Pier Ventreadb4ae62016-11-23 09:57:42 -0800332 String message = "getRouterIpv4 fails for device: " + deviceId + ".";
333 throw new DeviceConfigNotFoundException(message);
334 }
335 }
336
337 @Override
338 public Ip6Address getRouterIpv6(DeviceId deviceId) throws DeviceConfigNotFoundException {
339 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
340 if (srinfo != null) {
341 log.trace("getRouterIpv6 for device{} is {}", deviceId, srinfo.ipv6Loopback);
342 return srinfo.ipv6Loopback;
343 } else {
344 String message = "getRouterIpv6 fails for device: " + deviceId + ".";
Charles Chan319d1a22015-11-03 10:42:14 -0800345 throw new DeviceConfigNotFoundException(message);
sangho80f11cb2015-04-01 13:05:26 -0700346 }
sangho80f11cb2015-04-01 13:05:26 -0700347 }
348
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700349 @Override
Charles Chan319d1a22015-11-03 10:42:14 -0800350 public boolean isEdgeDevice(DeviceId deviceId) throws DeviceConfigNotFoundException {
Saurav Das7c305372015-10-28 12:39:42 -0700351 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
352 if (srinfo != null) {
353 log.trace("isEdgeDevice for device{} is {}", deviceId, srinfo.isEdge);
354 return srinfo.isEdge;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700355 } else {
Charles Chan319d1a22015-11-03 10:42:14 -0800356 String message = "isEdgeDevice fails for device: " + deviceId + ".";
357 throw new DeviceConfigNotFoundException(message);
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700358 }
359 }
360
sangho80f11cb2015-04-01 13:05:26 -0700361 @Override
362 public List<Integer> getAllDeviceSegmentIds() {
363 return allSegmentIds;
364 }
365
Charles Chan77277672015-10-20 16:24:19 -0700366 @Override
Pier Ventreb6a7f342016-11-26 21:05:22 -0800367 public Map<IpPrefix, List<PortNumber>> getSubnetPortsMap(DeviceId deviceId)
Saurav Das52d4ed72016-03-28 19:00:18 -0700368 throws DeviceConfigNotFoundException {
369 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
370 if (srinfo == null) {
371 String message = "getSubnetPortsMap fails for device: " + deviceId + ".";
372 throw new DeviceConfigNotFoundException(message);
373 }
Charles Chan77277672015-10-20 16:24:19 -0700374 // Construct subnet-port mapping from port-subnet mapping
Pier Ventreb6a7f342016-11-26 21:05:22 -0800375 SetMultimap<PortNumber, IpPrefix> portSubnetMap = srinfo.subnets;
376 Map<IpPrefix, List<PortNumber>> subnetPortMap = new HashMap<>();
Charles Chan82ab1932016-01-30 23:22:37 -0800377
378 portSubnetMap.entries().forEach(entry -> {
379 PortNumber port = entry.getKey();
Pier Ventreb6a7f342016-11-26 21:05:22 -0800380 IpPrefix subnet = entry.getValue();
Charles Chan82ab1932016-01-30 23:22:37 -0800381
Pier Ventreb6a7f342016-11-26 21:05:22 -0800382 if (subnet.prefixLength() == IpPrefix.MAX_INET_MASK_LENGTH ||
383 subnet.prefixLength() == IpPrefix.MAX_INET6_MASK_LENGTH) {
Charles Chanbbd004c2016-02-16 23:14:49 -0800384 return;
385 }
386
Charles Chan77277672015-10-20 16:24:19 -0700387 if (subnetPortMap.containsKey(subnet)) {
388 subnetPortMap.get(subnet).add(port);
389 } else {
390 ArrayList<PortNumber> ports = new ArrayList<>();
391 ports.add(port);
392 subnetPortMap.put(subnet, ports);
393 }
394 });
Charles Chan77277672015-10-20 16:24:19 -0700395 return subnetPortMap;
396 }
397
sangho80f11cb2015-04-01 13:05:26 -0700398 /**
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700399 * Returns the device identifier or data plane identifier (dpid)
400 * of a segment router given its segment id.
sangho80f11cb2015-04-01 13:05:26 -0700401 *
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700402 * @param sid segment id
403 * @return deviceId device identifier
sangho80f11cb2015-04-01 13:05:26 -0700404 */
405 public DeviceId getDeviceId(int sid) {
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700406 for (Map.Entry<DeviceId, SegmentRouterInfo> entry:
407 deviceConfigMap.entrySet()) {
Pier Ventreadb4ae62016-11-23 09:57:42 -0800408 if (entry.getValue().ipv4NodeSid == sid ||
409 entry.getValue().ipv6NodeSid == sid) {
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700410 return entry.getValue().deviceId;
sangho80f11cb2015-04-01 13:05:26 -0700411 }
412 }
413
414 return null;
415 }
416
417 /**
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700418 * Returns the device identifier or data plane identifier (dpid)
419 * of a segment router given its router ip address.
sangho80f11cb2015-04-01 13:05:26 -0700420 *
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700421 * @param ipAddress router ip address
422 * @return deviceId device identifier
sangho80f11cb2015-04-01 13:05:26 -0700423 */
424 public DeviceId getDeviceId(Ip4Address ipAddress) {
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700425 for (Map.Entry<DeviceId, SegmentRouterInfo> entry:
426 deviceConfigMap.entrySet()) {
Pier Ventreadb4ae62016-11-23 09:57:42 -0800427 if (entry.getValue().ipv4Loopback.equals(ipAddress)) {
428 return entry.getValue().deviceId;
429 }
430 }
431
432 return null;
433 }
434
435 /**
436 * Returns the device identifier or data plane identifier (dpid)
437 * of a segment router given its router ipv6 address.
438 *
439 * @param ipAddress router ipv6 address
440 * @return deviceId device identifier
441 */
442 public DeviceId getDeviceId(Ip6Address ipAddress) {
443 for (Map.Entry<DeviceId, SegmentRouterInfo> entry:
444 deviceConfigMap.entrySet()) {
445 if (entry.getValue().ipv6Loopback.equals(ipAddress)) {
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700446 return entry.getValue().deviceId;
sangho80f11cb2015-04-01 13:05:26 -0700447 }
448 }
449
450 return null;
451 }
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700452
453 /**
Saurav Das9f1c42e2015-10-23 10:51:11 -0700454 * Returns the configured port ip addresses for a segment router.
455 * These addresses serve as gateway IP addresses for the subnets configured
456 * on those ports.
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700457 *
458 * @param deviceId device identifier
Saurav Dasc28b3432015-10-30 17:45:38 -0700459 * @return immutable set of ip addresses configured on the ports or null if not found
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700460 */
Pier Ventreb6a7f342016-11-26 21:05:22 -0800461 public Set<IpAddress> getPortIPs(DeviceId deviceId) {
Saurav Das7c305372015-10-28 12:39:42 -0700462 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
463 if (srinfo != null) {
464 log.trace("getSubnetGatewayIps for device{} is {}", deviceId,
465 srinfo.gatewayIps.values());
Saurav Dasc28b3432015-10-30 17:45:38 -0700466 return ImmutableSet.copyOf(srinfo.gatewayIps.values());
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700467 }
Saurav Das7c305372015-10-28 12:39:42 -0700468 return null;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700469 }
470
471 /**
Charles Chancf789822019-03-22 10:04:27 -0700472 * Returns configured subnets for a segment router.
473 *
474 * @param deviceId device identifier
475 * @return list of ip prefixes or null if not found
476 */
477 public Set<IpPrefix> getConfiguredSubnets(DeviceId deviceId) {
478 Set<IpPrefix> subnets = srManager.interfaceService.getInterfaces().stream()
479 .filter(intf -> Objects.equals(deviceId, intf.connectPoint().deviceId()))
480 .flatMap(intf -> intf.ipAddressesList().stream())
481 .map(InterfaceIpAddress::subnetAddress)
482 .collect(Collectors.toSet());
483
484 if (subnets.isEmpty()) {
485 log.debug(NO_SUBNET, deviceId);
486 return Collections.emptySet();
487 }
488 return subnets;
489 }
490
491 /**
492 * Returns all subnets for a segment router, including subnets learnt from route service.
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700493 *
494 * @param deviceId device identifier
Saurav Das7c305372015-10-28 12:39:42 -0700495 * @return list of ip prefixes or null if not found
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700496 */
Pier Ventreb6a7f342016-11-26 21:05:22 -0800497 public Set<IpPrefix> getSubnets(DeviceId deviceId) {
Saurav Das7c305372015-10-28 12:39:42 -0700498 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
Charles Chan5eb66042018-06-18 14:42:17 -0700499 if (srinfo != null && srinfo.subnets != null) {
500 // Note: ImmutableSet.Builder.addAll calls the iterator of parameter internally,
501 // which is not protected by SynchronizedCollection mutex.
Pier Ventreb6a7f342016-11-26 21:05:22 -0800502 ImmutableSet.Builder<IpPrefix> builder = ImmutableSet.builder();
Charles Chan5eb66042018-06-18 14:42:17 -0700503 srinfo.subnets.forEach((k, v) -> builder.add(v));
504 return builder.build();
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700505 }
Saurav Das7c305372015-10-28 12:39:42 -0700506 return null;
507 }
508
Charles Chan46fdfaf2016-11-09 20:51:44 -0800509
Saurav Das7c305372015-10-28 12:39:42 -0700510 /**
Charles Chan46fdfaf2016-11-09 20:51:44 -0800511 * Returns the subnet configuration of given device and port.
Saurav Das7c305372015-10-28 12:39:42 -0700512 *
Charles Chan46fdfaf2016-11-09 20:51:44 -0800513 * @param deviceId Device ID
514 * @param port Port number
Pier Ventreb6a7f342016-11-26 21:05:22 -0800515 * @return The subnets configured on given port or empty set if
Charles Chan2f4d2bc2017-04-24 16:21:01 -0700516 * the port is unconfigured or suppressed.
Saurav Das7c305372015-10-28 12:39:42 -0700517 */
Pier Ventreb6a7f342016-11-26 21:05:22 -0800518 public Set<IpPrefix> getPortSubnets(DeviceId deviceId, PortNumber port) {
Charles Chan46fdfaf2016-11-09 20:51:44 -0800519 ConnectPoint connectPoint = new ConnectPoint(deviceId, port);
520
521 if (isSuppressedPort(connectPoint)) {
Pier Ventreb6a7f342016-11-26 21:05:22 -0800522 return Collections.emptySet();
Saurav Das7c305372015-10-28 12:39:42 -0700523 }
Charles Chan46fdfaf2016-11-09 20:51:44 -0800524
Charles Chancf789822019-03-22 10:04:27 -0700525 Set<IpPrefix> subnets = srManager.interfaceService.getInterfacesByPort(connectPoint).stream()
526 .flatMap(intf -> intf.ipAddressesList().stream())
527 .map(InterfaceIpAddress::subnetAddress)
528 .collect(Collectors.toSet());
Charles Chan46fdfaf2016-11-09 20:51:44 -0800529
Jon Hall31d84782017-01-18 20:15:44 -0800530 if (subnets.isEmpty()) {
Saurav Dasf9332192017-02-18 14:05:44 -0800531 log.debug(NO_SUBNET, connectPoint);
Pier Ventreb6a7f342016-11-26 21:05:22 -0800532 return Collections.emptySet();
Charles Chan46fdfaf2016-11-09 20:51:44 -0800533 }
Charles Chan2f4d2bc2017-04-24 16:21:01 -0700534 return subnets;
Pier Ventreb6a7f342016-11-26 21:05:22 -0800535 }
536
537 /**
Charles Chan873661e2017-11-30 15:37:50 -0800538 * Returns all ports that has a subnet that contains any of the given IP addresses.
539 *
540 * @param ips a set of IP addresses
541 * @return a set of connect point that has a subnet that contains any of the given IP addresses
542 */
543 public Set<ConnectPoint> getPortByIps(Set<IpAddress> ips) {
544 return srManager.interfaceService.getInterfaces().stream()
545 .filter(intf -> intf.ipAddressesList().stream().anyMatch(intfAddress ->
546 ips.stream().anyMatch(ip -> intfAddress.subnetAddress().contains(ip))))
547 .map(Interface::connectPoint)
548 .collect(Collectors.toSet());
549 }
550
551 /**
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700552 * Returns the router ip address of segment router that has the
553 * specified ip address in its subnets.
554 *
555 * @param destIpAddress target ip address
556 * @return router ip address
557 */
558 public Ip4Address getRouterIpAddressForASubnetHost(Ip4Address destIpAddress) {
Charles Chan70661362016-12-09 12:54:49 -0800559 Interface matchIntf = srManager.interfaceService.getMatchingInterface(destIpAddress);
560
561 if (matchIntf == null) {
562 log.debug("No router was found for {}", destIpAddress);
563 return null;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700564 }
565
Charles Chan70661362016-12-09 12:54:49 -0800566 DeviceId routerDeviceId = matchIntf.connectPoint().deviceId();
567 SegmentRouterInfo srInfo = deviceConfigMap.get(routerDeviceId);
568 if (srInfo == null) {
569 log.debug("No device config was found for {}", routerDeviceId);
570 return null;
571 }
572
Pier Ventreadb4ae62016-11-23 09:57:42 -0800573 return srInfo.ipv4Loopback;
574 }
575
576 /**
577 * Returns the router ipv6 address of segment router that has the
578 * specified ip address in its subnets.
579 *
580 * @param destIpAddress target ip address
581 * @return router ip address
582 */
583 public Ip6Address getRouterIpAddressForASubnetHost(Ip6Address destIpAddress) {
584 Interface matchIntf = srManager.interfaceService.getMatchingInterface(destIpAddress);
585
586 if (matchIntf == null) {
587 log.debug("No router was found for {}", destIpAddress);
588 return null;
589 }
590
591 DeviceId routerDeviceId = matchIntf.connectPoint().deviceId();
592 SegmentRouterInfo srInfo = deviceConfigMap.get(routerDeviceId);
593 if (srInfo == null) {
594 log.debug("No device config was found for {}", routerDeviceId);
595 return null;
596 }
597
598 return srInfo.ipv6Loopback;
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700599 }
600
601 /**
602 * Returns the router mac address of segment router that has the
603 * specified ip address as one of its subnet gateway ip address.
604 *
605 * @param gatewayIpAddress router gateway ip address
Saurav Das7c305372015-10-28 12:39:42 -0700606 * @return router mac address or null if not found
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700607 */
Pier Ventreb6a7f342016-11-26 21:05:22 -0800608 public MacAddress getRouterMacForAGatewayIp(IpAddress gatewayIpAddress) {
Srikanth Vavilapalli37a461b2015-04-07 15:12:32 -0700609 for (Map.Entry<DeviceId, SegmentRouterInfo> entry:
610 deviceConfigMap.entrySet()) {
611 if (entry.getValue().gatewayIps.
612 values().contains(gatewayIpAddress)) {
613 return entry.getValue().mac;
614 }
615 }
616
617 log.debug("Cannot find a router for {}", gatewayIpAddress);
618 return null;
619 }
sangho9b169e32015-04-14 16:27:13 -0700620
sangho9b169e32015-04-14 16:27:13 -0700621 /**
Charles Chan35f21e42017-06-26 18:30:18 -0700622 * Checks if the host IP is in any of the subnet defined in the router with the
sangho9b169e32015-04-14 16:27:13 -0700623 * device ID given.
624 *
625 * @param deviceId device identification of the router
626 * @param hostIp host IP address to check
Charles Chan35f21e42017-06-26 18:30:18 -0700627 * @return true if the given IP is within any of the subnet defined in the router,
628 * false if no subnet is defined in the router or if the host is not
629 * within any subnet defined in the router
sangho9b169e32015-04-14 16:27:13 -0700630 */
Pier Ventreb6a7f342016-11-26 21:05:22 -0800631 public boolean inSameSubnet(DeviceId deviceId, IpAddress hostIp) {
Charles Chancf789822019-03-22 10:04:27 -0700632 Set<IpPrefix> subnets = getConfiguredSubnets(deviceId);
sangho9b169e32015-04-14 16:27:13 -0700633 if (subnets == null) {
634 return false;
635 }
636
Pier Ventreb6a7f342016-11-26 21:05:22 -0800637 for (IpPrefix subnet: subnets) {
Charles Chan82ab1932016-01-30 23:22:37 -0800638 // Exclude /0 since it is a special case used for default route
639 if (subnet.prefixLength() != 0 && subnet.contains(hostIp)) {
sangho9b169e32015-04-14 16:27:13 -0700640 return true;
641 }
642 }
sangho9b169e32015-04-14 16:27:13 -0700643 return false;
644 }
sangho27462c62015-05-14 00:39:53 -0700645
646 /**
Charles Chandebfea32016-10-24 14:52:01 -0700647 * Checks if the IP is in the subnet defined on given connect point.
648 *
649 * @param connectPoint Connect point
650 * @param ip The IP address to check
651 * @return True if the IP belongs to the subnet.
652 * False if the IP does not belong to the subnet, or
653 * there is no subnet configuration on given connect point.
654 */
655 public boolean inSameSubnet(ConnectPoint connectPoint, IpAddress ip) {
Charles Chan35f21e42017-06-26 18:30:18 -0700656 return getPortSubnets(connectPoint.deviceId(), connectPoint.port()).stream()
657 .anyMatch(ipPrefix -> ipPrefix.contains(ip));
Charles Chandebfea32016-10-24 14:52:01 -0700658 }
659
660 /**
sangho27462c62015-05-14 00:39:53 -0700661 * Returns the ports corresponding to the adjacency Sid given.
662 *
663 * @param deviceId device identification of the router
664 * @param sid adjacency Sid
Charles Chan9bec8a32015-12-01 10:00:51 -0800665 * @return set of port numbers
sangho27462c62015-05-14 00:39:53 -0700666 */
Charles Chan9bec8a32015-12-01 10:00:51 -0800667 public Set<Integer> getPortsForAdjacencySid(DeviceId deviceId, int sid) {
Saurav Das7c305372015-10-28 12:39:42 -0700668 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
Charles Chan9bec8a32015-12-01 10:00:51 -0800669 return srinfo != null ?
670 ImmutableSet.copyOf(srinfo.adjacencySids.get(sid)) :
671 ImmutableSet.copyOf(new HashSet<>());
sangho27462c62015-05-14 00:39:53 -0700672 }
673
674 /**
675 * Check if the Sid given is whether adjacency Sid of the router device or not.
676 *
677 * @param deviceId device identification of the router
678 * @param sid Sid to check
679 * @return true if the Sid given is the adjacency Sid of the device,
680 * otherwise false
681 */
682 public boolean isAdjacencySid(DeviceId deviceId, int sid) {
Saurav Das7c305372015-10-28 12:39:42 -0700683 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
Charles Chan9bec8a32015-12-01 10:00:51 -0800684 return srinfo != null && srinfo.adjacencySids.containsKey(sid);
sangho27462c62015-05-14 00:39:53 -0700685 }
Charles Chan43547ca2016-02-10 20:46:58 -0800686
Charles Chanc91c8782016-03-30 17:54:24 -0700687 /**
Charles Chanc22cef32016-04-29 14:38:22 -0700688 * Add subnet to specific connect point.
689 *
690 * @param cp connect point
Pier Ventreb6a7f342016-11-26 21:05:22 -0800691 * @param ipPrefix subnet being added to the device
Charles Chanc22cef32016-04-29 14:38:22 -0700692 */
Pier Ventreb6a7f342016-11-26 21:05:22 -0800693 public void addSubnet(ConnectPoint cp, IpPrefix ipPrefix) {
Charles Chanc22cef32016-04-29 14:38:22 -0700694 checkNotNull(cp);
Pier Ventreb6a7f342016-11-26 21:05:22 -0800695 checkNotNull(ipPrefix);
Charles Chanc22cef32016-04-29 14:38:22 -0700696 SegmentRouterInfo srinfo = deviceConfigMap.get(cp.deviceId());
697 if (srinfo == null) {
698 log.warn("Device {} is not configured. Abort.", cp.deviceId());
699 return;
700 }
Pier Ventreb6a7f342016-11-26 21:05:22 -0800701 srinfo.subnets.put(cp.port(), ipPrefix);
Charles Chanc22cef32016-04-29 14:38:22 -0700702 }
703
704 /**
705 * Remove subnet from specific connect point.
706 *
707 * @param cp connect point
Pier Ventreb6a7f342016-11-26 21:05:22 -0800708 * @param ipPrefix subnet being removed to the device
Charles Chanc22cef32016-04-29 14:38:22 -0700709 */
Pier Ventreb6a7f342016-11-26 21:05:22 -0800710 public void removeSubnet(ConnectPoint cp, IpPrefix ipPrefix) {
Charles Chanc22cef32016-04-29 14:38:22 -0700711 checkNotNull(cp);
Pier Ventreb6a7f342016-11-26 21:05:22 -0800712 checkNotNull(ipPrefix);
Charles Chanc22cef32016-04-29 14:38:22 -0700713 SegmentRouterInfo srinfo = deviceConfigMap.get(cp.deviceId());
714 if (srinfo == null) {
715 log.warn("Device {} is not configured. Abort.", cp.deviceId());
716 return;
717 }
Pier Ventreb6a7f342016-11-26 21:05:22 -0800718 srinfo.subnets.remove(cp.port(), ipPrefix);
Charles Chanc22cef32016-04-29 14:38:22 -0700719 }
Charles Chan46fdfaf2016-11-09 20:51:44 -0800720
721 private boolean isSuppressedPort(ConnectPoint connectPoint) {
722 SegmentRoutingAppConfig appConfig = srManager.cfgService
Ray Milkeyb85de082017-04-05 09:42:04 -0700723 .getConfig(srManager.appId(), SegmentRoutingAppConfig.class);
Charles Chan46fdfaf2016-11-09 20:51:44 -0800724 if (appConfig != null && appConfig.suppressSubnet().contains(connectPoint)) {
Charles Chandebfea32016-10-24 14:52:01 -0700725 log.info("Interface configuration on port {} is ignored", connectPoint);
Charles Chan46fdfaf2016-11-09 20:51:44 -0800726 return true;
727 }
728 return false;
729 }
Saurav Das261c3002017-06-13 15:35:54 -0700730
731 public boolean isPairedEdge(DeviceId deviceId) throws DeviceConfigNotFoundException {
732 if (!isEdgeDevice(deviceId)) {
733 return false;
734 }
735 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
736 return (srinfo.pairDeviceId == null) ? false : true;
737 }
738
739 public DeviceId getPairDeviceId(DeviceId deviceId) throws DeviceConfigNotFoundException {
740 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
741 if (srinfo != null) {
742 return srinfo.pairDeviceId;
743 } else {
744 String message = "getPairDeviceId fails for device: " + deviceId + ".";
745 throw new DeviceConfigNotFoundException(message);
746 }
747 }
748
749 public PortNumber getPairLocalPort(DeviceId deviceId)
750 throws DeviceConfigNotFoundException {
751 SegmentRouterInfo srinfo = deviceConfigMap.get(deviceId);
752 if (srinfo != null) {
753 return srinfo.pairLocalPort;
754 } else {
755 String message = "getPairLocalPort fails for device: " + deviceId + ".";
756 throw new DeviceConfigNotFoundException(message);
757 }
758 }
759
Saurav Dasec683dc2018-04-27 18:42:30 -0700760 public boolean isPairLocalPort(DeviceId devId, PortNumber pnum) {
761 return pnum.equals(srManager.getPairLocalPort(devId).orElse(null));
762 }
Jonathan Hart8fa9ec52016-02-16 10:30:37 -0800763}