blob: e6fa300e53a7d8e9c4a053d52ab9a7c5be08f8db [file] [log] [blame]
Thomas Vachuska4f1a60c2014-10-28 13:39:07 -07001/*
2 * Copyright 2014 Open Networking Laboratory
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
Brian O'Connorabafb502014-12-02 22:26:20 -080016package org.onosproject.net.link.impl;
tomdb0d03f2014-08-27 16:34:15 -070017
Simon Huntff663742015-05-14 13:33:05 -070018import com.google.common.base.Predicate;
19import com.google.common.collect.FluentIterable;
20import com.google.common.collect.Sets;
tomdb0d03f2014-08-27 16:34:15 -070021import org.apache.felix.scr.annotations.Activate;
22import org.apache.felix.scr.annotations.Component;
23import org.apache.felix.scr.annotations.Deactivate;
tom5f38b3a2014-08-27 23:50:54 -070024import org.apache.felix.scr.annotations.Reference;
25import org.apache.felix.scr.annotations.ReferenceCardinality;
tomdb0d03f2014-08-27 16:34:15 -070026import org.apache.felix.scr.annotations.Service;
Changhoon Yoon541ef712015-05-23 17:18:34 +090027import org.onosproject.core.Permission;
Brian O'Connorabafb502014-12-02 22:26:20 -080028import org.onosproject.event.EventDeliveryService;
Simon Huntff663742015-05-14 13:33:05 -070029import org.onosproject.event.ListenerRegistry;
Sahil Lele3a0cdd52015-07-21 14:16:31 -070030import org.onosproject.incubator.net.config.NetworkConfigEvent;
31import org.onosproject.incubator.net.config.NetworkConfigListener;
32import org.onosproject.incubator.net.config.NetworkConfigService;
33import org.onosproject.incubator.net.config.basics.BasicLinkConfig;
Brian O'Connorabafb502014-12-02 22:26:20 -080034import org.onosproject.net.ConnectPoint;
Sahil Lele3a0cdd52015-07-21 14:16:31 -070035import org.onosproject.net.DefaultAnnotations;
Brian O'Connorabafb502014-12-02 22:26:20 -080036import org.onosproject.net.DeviceId;
37import org.onosproject.net.Link;
38import org.onosproject.net.Link.State;
Sahil Lele3a0cdd52015-07-21 14:16:31 -070039import org.onosproject.net.LinkKey;
Brian O'Connorabafb502014-12-02 22:26:20 -080040import org.onosproject.net.MastershipRole;
Sahil Lele3a0cdd52015-07-21 14:16:31 -070041import org.onosproject.net.SparseAnnotations;
Brian O'Connorabafb502014-12-02 22:26:20 -080042import org.onosproject.net.device.DeviceEvent;
43import org.onosproject.net.device.DeviceListener;
44import org.onosproject.net.device.DeviceService;
Sahil Lele3a0cdd52015-07-21 14:16:31 -070045import org.onosproject.net.link.DefaultLinkDescription;
Brian O'Connorabafb502014-12-02 22:26:20 -080046import org.onosproject.net.link.LinkAdminService;
47import org.onosproject.net.link.LinkDescription;
48import org.onosproject.net.link.LinkEvent;
49import org.onosproject.net.link.LinkListener;
50import org.onosproject.net.link.LinkProvider;
51import org.onosproject.net.link.LinkProviderRegistry;
52import org.onosproject.net.link.LinkProviderService;
53import org.onosproject.net.link.LinkService;
54import org.onosproject.net.link.LinkStore;
55import org.onosproject.net.link.LinkStoreDelegate;
56import org.onosproject.net.provider.AbstractProviderRegistry;
57import org.onosproject.net.provider.AbstractProviderService;
tomdb0d03f2014-08-27 16:34:15 -070058import org.slf4j.Logger;
tom5f38b3a2014-08-27 23:50:54 -070059
Sahil Lele3a0cdd52015-07-21 14:16:31 -070060import java.time.Duration;
Simon Huntff663742015-05-14 13:33:05 -070061import java.util.Set;
tomdb0d03f2014-08-27 16:34:15 -070062
Ray Milkey7bbeb3f2014-12-11 14:59:26 -080063import static com.google.common.base.Preconditions.checkNotNull;
Sahil Lele3a0cdd52015-07-21 14:16:31 -070064import static com.google.common.base.Preconditions.checkState;
65import static org.onosproject.net.LinkKey.linkKey;
Ray Milkey7bbeb3f2014-12-11 14:59:26 -080066import static org.slf4j.LoggerFactory.getLogger;
Changhoon Yoon541ef712015-05-23 17:18:34 +090067import static org.onosproject.security.AppGuard.checkPermission;
68
Ray Milkey7bbeb3f2014-12-11 14:59:26 -080069
tomdb0d03f2014-08-27 16:34:15 -070070/**
71 * Provides basic implementation of the link SB & NB APIs.
72 */
73@Component(immediate = true)
74@Service
tom35c0dc32014-09-19 10:00:58 -070075public class LinkManager
tomdc361b62014-09-09 20:36:52 -070076 extends AbstractProviderRegistry<LinkProvider, LinkProviderService>
77 implements LinkService, LinkAdminService, LinkProviderRegistry {
tomeadbb462014-09-07 16:10:19 -070078
79 private static final String DEVICE_ID_NULL = "Device ID cannot be null";
80 private static final String LINK_DESC_NULL = "Link description cannot be null";
81 private static final String CONNECT_POINT_NULL = "Connection point cannot be null";
tomdb0d03f2014-08-27 16:34:15 -070082
tom5f38b3a2014-08-27 23:50:54 -070083 private final Logger log = getLogger(getClass());
tomdb0d03f2014-08-27 16:34:15 -070084
Simon Huntff663742015-05-14 13:33:05 -070085 protected final ListenerRegistry<LinkEvent, LinkListener>
86 listenerRegistry = new ListenerRegistry<>();
tom5f38b3a2014-08-27 23:50:54 -070087
alshabibb5522ff2014-09-29 19:20:00 -070088 private final LinkStoreDelegate delegate = new InternalStoreDelegate();
tomc78acee2014-09-24 15:16:55 -070089
90 private final DeviceListener deviceListener = new InternalDeviceListener();
tom89b63c52014-09-16 09:19:51 -070091
Sahil Lele3a0cdd52015-07-21 14:16:31 -070092 private final NetworkConfigListener networkConfigListener = new InternalNetworkConfigListener();
93
tom89b63c52014-09-16 09:19:51 -070094 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
tom35c0dc32014-09-19 10:00:58 -070095 protected LinkStore store;
96
97 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
tom89b63c52014-09-16 09:19:51 -070098 protected DeviceService deviceService;
tom4c6606f2014-09-07 11:11:21 -070099
tom5f38b3a2014-08-27 23:50:54 -0700100 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
tom0d395262014-09-07 16:53:40 -0700101 protected EventDeliveryService eventDispatcher;
tomdb0d03f2014-08-27 16:34:15 -0700102
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700103 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
104 protected NetworkConfigService networkConfigService;
105
tomdb0d03f2014-08-27 16:34:15 -0700106 @Activate
107 public void activate() {
tomc78acee2014-09-24 15:16:55 -0700108 store.setDelegate(delegate);
tom96dfcab2014-08-28 09:26:03 -0700109 eventDispatcher.addSink(LinkEvent.class, listenerRegistry);
tom89b63c52014-09-16 09:19:51 -0700110 deviceService.addListener(deviceListener);
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700111 networkConfigService.addListener(networkConfigListener);
tomdb0d03f2014-08-27 16:34:15 -0700112 log.info("Started");
113 }
114
115 @Deactivate
116 public void deactivate() {
tomc78acee2014-09-24 15:16:55 -0700117 store.unsetDelegate(delegate);
tom5f38b3a2014-08-27 23:50:54 -0700118 eventDispatcher.removeSink(LinkEvent.class);
tom89b63c52014-09-16 09:19:51 -0700119 deviceService.removeListener(deviceListener);
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700120 networkConfigService.removeListener(networkConfigListener);
tomdb0d03f2014-08-27 16:34:15 -0700121 log.info("Stopped");
122 }
123
124 @Override
tomeadbb462014-09-07 16:10:19 -0700125 public int getLinkCount() {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900126 checkPermission(Permission.LINK_READ);
127
tomeadbb462014-09-07 16:10:19 -0700128 return store.getLinkCount();
129 }
130
131 @Override
132 public Iterable<Link> getLinks() {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900133 checkPermission(Permission.LINK_READ);
134
tomeadbb462014-09-07 16:10:19 -0700135 return store.getLinks();
136 }
137
138 @Override
Yuta HIGUCHIf1f2ac02014-11-26 14:02:22 -0800139 public Iterable<Link> getActiveLinks() {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900140 checkPermission(Permission.LINK_READ);
141
Yuta HIGUCHIf1f2ac02014-11-26 14:02:22 -0800142 return FluentIterable.from(getLinks())
143 .filter(new Predicate<Link>() {
144
145 @Override
146 public boolean apply(Link input) {
147 return input.state() == State.ACTIVE;
148 }
149 });
150 }
151
152 @Override
tomeadbb462014-09-07 16:10:19 -0700153 public Set<Link> getDeviceLinks(DeviceId deviceId) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900154 checkPermission(Permission.LINK_READ);
155
tomeadbb462014-09-07 16:10:19 -0700156 checkNotNull(deviceId, DEVICE_ID_NULL);
157 return Sets.union(store.getDeviceEgressLinks(deviceId),
tomdc361b62014-09-09 20:36:52 -0700158 store.getDeviceIngressLinks(deviceId));
tomeadbb462014-09-07 16:10:19 -0700159 }
160
161 @Override
162 public Set<Link> getDeviceEgressLinks(DeviceId deviceId) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900163 checkPermission(Permission.LINK_READ);
164
tomeadbb462014-09-07 16:10:19 -0700165 checkNotNull(deviceId, DEVICE_ID_NULL);
166 return store.getDeviceEgressLinks(deviceId);
167 }
168
169 @Override
tomd176fc42014-09-08 00:12:30 -0700170 public Set<Link> getDeviceIngressLinks(DeviceId deviceId) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900171 checkPermission(Permission.LINK_READ);
172
tomeadbb462014-09-07 16:10:19 -0700173 checkNotNull(deviceId, DEVICE_ID_NULL);
174 return store.getDeviceIngressLinks(deviceId);
175 }
176
177 @Override
178 public Set<Link> getLinks(ConnectPoint connectPoint) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900179 checkPermission(Permission.LINK_READ);
180
tomeadbb462014-09-07 16:10:19 -0700181 checkNotNull(connectPoint, CONNECT_POINT_NULL);
182 return Sets.union(store.getEgressLinks(connectPoint),
tomdc361b62014-09-09 20:36:52 -0700183 store.getIngressLinks(connectPoint));
tomeadbb462014-09-07 16:10:19 -0700184 }
185
186 @Override
187 public Set<Link> getEgressLinks(ConnectPoint connectPoint) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900188 checkPermission(Permission.LINK_READ);
189
tomeadbb462014-09-07 16:10:19 -0700190 checkNotNull(connectPoint, CONNECT_POINT_NULL);
191 return store.getEgressLinks(connectPoint);
192 }
193
194 @Override
tomd176fc42014-09-08 00:12:30 -0700195 public Set<Link> getIngressLinks(ConnectPoint connectPoint) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900196 checkPermission(Permission.LINK_READ);
197
tomeadbb462014-09-07 16:10:19 -0700198 checkNotNull(connectPoint, CONNECT_POINT_NULL);
199 return store.getIngressLinks(connectPoint);
200 }
201
202 @Override
tomd176fc42014-09-08 00:12:30 -0700203 public Link getLink(ConnectPoint src, ConnectPoint dst) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900204 checkPermission(Permission.LINK_READ);
205
tomeadbb462014-09-07 16:10:19 -0700206 checkNotNull(src, CONNECT_POINT_NULL);
207 checkNotNull(dst, CONNECT_POINT_NULL);
tomd176fc42014-09-08 00:12:30 -0700208 return store.getLink(src, dst);
tomeadbb462014-09-07 16:10:19 -0700209 }
210
211 @Override
212 public void removeLinks(ConnectPoint connectPoint) {
Ayaka Koshibeb5c63a02014-10-18 18:42:27 -0700213 if (deviceService.getRole(connectPoint.deviceId()) != MastershipRole.MASTER) {
214 return;
215 }
Thomas Vachuska57126fe2014-11-11 17:13:24 -0800216 removeLinks(getLinks(connectPoint), false);
tomeadbb462014-09-07 16:10:19 -0700217 }
218
219 @Override
220 public void removeLinks(DeviceId deviceId) {
Ayaka Koshibeb5c63a02014-10-18 18:42:27 -0700221 if (deviceService.getRole(deviceId) != MastershipRole.MASTER) {
222 return;
223 }
Thomas Vachuska57126fe2014-11-11 17:13:24 -0800224 removeLinks(getDeviceLinks(deviceId), false);
tomeadbb462014-09-07 16:10:19 -0700225 }
226
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700227 public void removeLink(ConnectPoint src, ConnectPoint dst) {
228 post(store.removeLink(src, dst));
229 }
230
tomeadbb462014-09-07 16:10:19 -0700231 @Override
232 public void addListener(LinkListener listener) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900233 checkPermission(Permission.LINK_EVENT);
tomeadbb462014-09-07 16:10:19 -0700234 listenerRegistry.addListener(listener);
235 }
236
237 @Override
238 public void removeListener(LinkListener listener) {
Changhoon Yoon541ef712015-05-23 17:18:34 +0900239 checkPermission(Permission.LINK_EVENT);
tomeadbb462014-09-07 16:10:19 -0700240 listenerRegistry.removeListener(listener);
241 }
242
tom89b63c52014-09-16 09:19:51 -0700243 // Auxiliary interceptor for device remove events to prune links that
244 // are associated with the removed device or its port.
tomc78acee2014-09-24 15:16:55 -0700245 private class InternalDeviceListener implements DeviceListener {
tom89b63c52014-09-16 09:19:51 -0700246 @Override
247 public void event(DeviceEvent event) {
248 if (event.type() == DeviceEvent.Type.DEVICE_REMOVED) {
249 removeLinks(event.subject().id());
250 } else if (event.type() == DeviceEvent.Type.PORT_REMOVED) {
251 removeLinks(new ConnectPoint(event.subject().id(),
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700252 event.port().number()));
tom89b63c52014-09-16 09:19:51 -0700253 }
254 }
255 }
256
tom7869ad92014-09-09 14:32:08 -0700257 @Override
258 protected LinkProviderService createProviderService(LinkProvider provider) {
259 return new InternalLinkProviderService(provider);
260 }
261
tomdb0d03f2014-08-27 16:34:15 -0700262 // Personalized link provider service issued to the supplied provider.
tomdc361b62014-09-09 20:36:52 -0700263 private class InternalLinkProviderService
264 extends AbstractProviderService<LinkProvider>
265 implements LinkProviderService {
tomdb0d03f2014-08-27 16:34:15 -0700266
tomcfde0622014-09-09 11:02:42 -0700267 InternalLinkProviderService(LinkProvider provider) {
tomdb0d03f2014-08-27 16:34:15 -0700268 super(provider);
269 }
270
271 @Override
272 public void linkDetected(LinkDescription linkDescription) {
tomeadbb462014-09-07 16:10:19 -0700273 checkNotNull(linkDescription, LINK_DESC_NULL);
274 checkValidity();
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700275 linkDescription = validateLink(linkDescription);
tomeadbb462014-09-07 16:10:19 -0700276 LinkEvent event = store.createOrUpdateLink(provider().id(),
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700277 linkDescription);
tomdc361b62014-09-09 20:36:52 -0700278 if (event != null) {
Thomas Vachuska0e752bd2014-10-22 22:33:41 -0700279 log.info("Link {} detected", linkDescription);
tomdc361b62014-09-09 20:36:52 -0700280 post(event);
281 }
tomdb0d03f2014-08-27 16:34:15 -0700282 }
283
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700284 // returns a LinkDescription made from the union of the BasicLinkConfig
285 // annotations if it exists
286 private LinkDescription validateLink(LinkDescription linkDescription) {
287 // TODO Investigate whether this can be made more efficient
288 BasicLinkConfig cfg = networkConfigService.getConfig(linkKey(linkDescription.src(),
289 linkDescription.dst()),
290 BasicLinkConfig.class);
291 BasicLinkConfig cfgTwo = networkConfigService.getConfig(linkKey(linkDescription.dst(),
292 linkDescription.src()),
293 BasicLinkConfig.class);
294
295 checkState(cfg == null || cfg.isAllowed(), "Link " + linkDescription.toString() + " is not allowed");
296 checkState(cfgTwo == null || cfgTwo.isAllowed(), "Link " + linkDescription.toString() + " is not allowed");
297 if (cfg != null) {
298 SparseAnnotations finalSparse = processAnnotations(cfg, linkDescription);
299 // check whether config has a specified type
300 if (cfg.type() != Link.Type.DIRECT) {
301 linkDescription = new DefaultLinkDescription(linkDescription.src(),
302 linkDescription.dst(),
303 cfg.type(), finalSparse);
304 } else {
305 linkDescription = new DefaultLinkDescription(linkDescription.src(),
306 linkDescription.dst(),
307 linkDescription.type(), finalSparse);
308 }
309 }
310 return linkDescription;
311 }
312
313 // supplements or replaces linkDescriptions's annotations with BasicLinkConfig's
314 // annotations
315 private SparseAnnotations processAnnotations(BasicLinkConfig cfg, LinkDescription linkDescription) {
316 SparseAnnotations originalAnnotations = linkDescription.annotations();
317 DefaultAnnotations.Builder newBuilder = DefaultAnnotations.builder();
318 if (cfg.type() != Link.Type.DIRECT) {
319 newBuilder.set(cfg.TYPE, cfg.type().toString());
320 }
321 if (cfg.latency() != Duration.ofNanos(-1)) {
322 newBuilder.set(cfg.LATENCY, cfg.latency().toString());
323 }
324 if (cfg.bandwidth() != -1) {
325 newBuilder.set(cfg.BANDWIDTH, String.valueOf(cfg.bandwidth()));
326 }
327 DefaultAnnotations newAnnotations = newBuilder.build();
328 return DefaultAnnotations.union(originalAnnotations, newAnnotations);
329 }
330
tomdb0d03f2014-08-27 16:34:15 -0700331 @Override
332 public void linkVanished(LinkDescription linkDescription) {
tomeadbb462014-09-07 16:10:19 -0700333 checkNotNull(linkDescription, LINK_DESC_NULL);
334 checkValidity();
Ayaka Koshibeb5c63a02014-10-18 18:42:27 -0700335
336 ConnectPoint src = linkDescription.src();
337 ConnectPoint dst = linkDescription.dst();
alshabibdfc7afb2014-10-21 20:13:27 -0700338
Thomas Vachuska57126fe2014-11-11 17:13:24 -0800339 LinkEvent event = store.removeOrDownLink(src, dst);
tomdc361b62014-09-09 20:36:52 -0700340 if (event != null) {
341 log.info("Link {} vanished", linkDescription);
342 post(event);
343 }
tomeadbb462014-09-07 16:10:19 -0700344 }
345
346 @Override
347 public void linksVanished(ConnectPoint connectPoint) {
348 checkNotNull(connectPoint, "Connect point cannot be null");
349 checkValidity();
alshabib12288c82014-10-23 10:24:23 -0700350
Yuta HIGUCHIddb77722014-12-11 19:39:04 -0800351 log.debug("Links for connection point {} vanished", connectPoint);
Yuta HIGUCHIe794cbe2014-10-17 13:21:23 -0700352 // FIXME: This will remove links registered by other providers
Thomas Vachuska57126fe2014-11-11 17:13:24 -0800353 removeLinks(getLinks(connectPoint), true);
tomeadbb462014-09-07 16:10:19 -0700354 }
355
356 @Override
357 public void linksVanished(DeviceId deviceId) {
358 checkNotNull(deviceId, DEVICE_ID_NULL);
359 checkValidity();
alshabib12288c82014-10-23 10:24:23 -0700360
Yuta HIGUCHIddb77722014-12-11 19:39:04 -0800361 log.debug("Links for device {} vanished", deviceId);
Thomas Vachuska57126fe2014-11-11 17:13:24 -0800362 removeLinks(getDeviceLinks(deviceId), true);
tomdb0d03f2014-08-27 16:34:15 -0700363 }
364 }
tomeadbb462014-09-07 16:10:19 -0700365
366 // Removes all links in the specified set and emits appropriate events.
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700367 private void removeLinks(Set<Link> links, boolean isSoftRemove) {
tomeadbb462014-09-07 16:10:19 -0700368 for (Link link : links) {
Thomas Vachuska57126fe2014-11-11 17:13:24 -0800369 LinkEvent event = isSoftRemove ?
370 store.removeOrDownLink(link.src(), link.dst()) :
371 store.removeLink(link.src(), link.dst());
Yuta HIGUCHIddb77722014-12-11 19:39:04 -0800372 if (event != null) {
373 log.info("Link {} removed/vanished", event.subject());
374 post(event);
375 }
tomeadbb462014-09-07 16:10:19 -0700376 }
377 }
378
379 // Posts the specified event to the local event dispatcher.
380 private void post(LinkEvent event) {
tomdc361b62014-09-09 20:36:52 -0700381 if (event != null) {
tomeadbb462014-09-07 16:10:19 -0700382 eventDispatcher.post(event);
383 }
384 }
385
tomc78acee2014-09-24 15:16:55 -0700386 // Store delegate to re-post events emitted from the store.
387 private class InternalStoreDelegate implements LinkStoreDelegate {
388 @Override
389 public void notify(LinkEvent event) {
390 post(event);
391 }
392 }
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700393
394 // listens for NetworkConfigEvents of type BasicLinkConfig and removes
395 // links that the config does not allow
396 private class InternalNetworkConfigListener implements NetworkConfigListener {
397 @Override
398 public void event(NetworkConfigEvent event) {
399 if ((event.type() == NetworkConfigEvent.Type.CONFIG_ADDED ||
400 event.type() == NetworkConfigEvent.Type.CONFIG_UPDATED) &&
401 event.configClass().equals(BasicLinkConfig.class)) {
402 log.info("Detected Link network config event {}", event.type());
403 LinkKey lk = (LinkKey) event.subject();
404 BasicLinkConfig cfg = networkConfigService.getConfig(lk, BasicLinkConfig.class);
405 if (cfg != null && !cfg.isAllowed()) {
406 log.info("Kicking out links between {} and {}", lk.src(), lk.dst());
407 removeLink(lk.src(), lk.dst());
408 removeLink(lk.dst(), lk.src());
409 }
410 }
411 }
412 }
tomdb0d03f2014-08-27 16:34:15 -0700413}