blob: c1b8aa156bf00e2f2619bdfbc5d4d0ac256dc882 [file] [log] [blame]
Thomas Vachuska781d18b2014-10-27 10:31:25 -07001/*
Thomas Vachuska4f1a60c2014-10-28 13:39:07 -07002 * Copyright 2014 Open Networking Laboratory
Thomas Vachuska781d18b2014-10-27 10:31:25 -07003 *
Thomas Vachuska4f1a60c2014-10-28 13:39:07 -07004 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
Thomas Vachuska781d18b2014-10-27 10:31:25 -07007 *
Thomas Vachuska4f1a60c2014-10-28 13:39:07 -07008 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
Thomas Vachuska781d18b2014-10-27 10:31:25 -070015 */
Brian O'Connorabafb502014-12-02 22:26:20 -080016package org.onosproject.provider.lldp.impl;
alshabib7911a052014-10-16 17:49:37 -070017
Thomas Vachuska6f94ded2015-02-21 14:02:38 -080018import com.google.common.base.Strings;
19import com.google.common.collect.ImmutableMap;
20import com.google.common.collect.ImmutableSet;
alshabib7911a052014-10-16 17:49:37 -070021import org.apache.felix.scr.annotations.Activate;
22import org.apache.felix.scr.annotations.Component;
23import org.apache.felix.scr.annotations.Deactivate;
Yuta HIGUCHI41289382014-12-19 17:47:12 -080024import org.apache.felix.scr.annotations.Modified;
25import org.apache.felix.scr.annotations.Property;
alshabib7911a052014-10-16 17:49:37 -070026import org.apache.felix.scr.annotations.Reference;
27import org.apache.felix.scr.annotations.ReferenceCardinality;
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -080028import org.onlab.packet.Ethernet;
29import org.onosproject.core.ApplicationId;
30import org.onosproject.core.CoreService;
Brian O'Connorabafb502014-12-02 22:26:20 -080031import org.onosproject.mastership.MastershipEvent;
32import org.onosproject.mastership.MastershipListener;
33import org.onosproject.mastership.MastershipService;
34import org.onosproject.net.ConnectPoint;
35import org.onosproject.net.Device;
36import org.onosproject.net.DeviceId;
37import org.onosproject.net.Port;
38import org.onosproject.net.device.DeviceEvent;
39import org.onosproject.net.device.DeviceListener;
40import org.onosproject.net.device.DeviceService;
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -080041import org.onosproject.net.flow.DefaultTrafficSelector;
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -080042import org.onosproject.net.flow.TrafficSelector;
Brian O'Connorabafb502014-12-02 22:26:20 -080043import org.onosproject.net.link.LinkProvider;
44import org.onosproject.net.link.LinkProviderRegistry;
45import org.onosproject.net.link.LinkProviderService;
46import org.onosproject.net.packet.PacketContext;
Jonathan Hart3cfce8e2015-01-14 16:43:27 -080047import org.onosproject.net.packet.PacketPriority;
Brian O'Connorabafb502014-12-02 22:26:20 -080048import org.onosproject.net.packet.PacketProcessor;
49import org.onosproject.net.packet.PacketService;
50import org.onosproject.net.provider.AbstractProvider;
51import org.onosproject.net.provider.ProviderId;
Yuta HIGUCHI41289382014-12-19 17:47:12 -080052import org.osgi.service.component.ComponentContext;
alshabib7911a052014-10-16 17:49:37 -070053import org.slf4j.Logger;
54
Thomas Vachuska6f94ded2015-02-21 14:02:38 -080055import java.io.IOException;
56import java.util.Dictionary;
57import java.util.EnumSet;
58import java.util.Map;
59import java.util.concurrent.ConcurrentHashMap;
60import java.util.concurrent.ScheduledExecutorService;
61
62import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
63import static java.util.concurrent.TimeUnit.SECONDS;
64import static org.onlab.util.Tools.groupedThreads;
65import static org.slf4j.LoggerFactory.getLogger;
Yuta HIGUCHI41289382014-12-19 17:47:12 -080066
alshabib7911a052014-10-16 17:49:37 -070067
68/**
69 * Provider which uses an OpenFlow controller to detect network
70 * infrastructure links.
71 */
72@Component(immediate = true)
73public class LLDPLinkProvider extends AbstractProvider implements LinkProvider {
74
Yuta HIGUCHI41289382014-12-19 17:47:12 -080075 private static final String PROP_USE_BDDP = "useBDDP";
Saurav Dasc313c402015-02-27 10:09:47 -080076 private static final String PROP_DISABLE_LD = "disableLinkDiscovery";
Yuta HIGUCHI41289382014-12-19 17:47:12 -080077 private static final String PROP_LLDP_SUPPRESSION = "lldpSuppression";
78
79 private static final String DEFAULT_LLDP_SUPPRESSION_CONFIG = "../config/lldp_suppression.json";
80
alshabib7911a052014-10-16 17:49:37 -070081 private final Logger log = getLogger(getClass());
82
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -080083 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
84 protected CoreService coreService;
85
86 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
alshabib7911a052014-10-16 17:49:37 -070087 protected LinkProviderRegistry providerRegistry;
88
89 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
90 protected DeviceService deviceService;
91
92 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -080093 protected PacketService packetService;
alshabib7911a052014-10-16 17:49:37 -070094
alshabib875d6262014-10-17 16:19:40 -070095 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
96 protected MastershipService masterService;
97
alshabib7911a052014-10-16 17:49:37 -070098 private LinkProviderService providerService;
99
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800100 private ScheduledExecutorService executor;
101
Saurav Dasc313c402015-02-27 10:09:47 -0800102 @Property(name = PROP_USE_BDDP, label = "use BDDP for link discovery")
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800103 private boolean useBDDP = true;
alshabib7911a052014-10-16 17:49:37 -0700104
Saurav Dasc313c402015-02-27 10:09:47 -0800105 @Property(name = PROP_DISABLE_LD, label = "permanently disable link discovery")
106 private boolean disableLD = false;
107
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800108 private static final long INIT_DELAY = 5;
109 private static final long DELAY = 5;
alshabib7911a052014-10-16 17:49:37 -0700110
Saurav Dasc313c402015-02-27 10:09:47 -0800111 @Property(name = PROP_LLDP_SUPPRESSION,
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800112 label = "Path to LLDP suppression configuration file")
113 private String filePath = DEFAULT_LLDP_SUPPRESSION_CONFIG;
114
115
alshabib7911a052014-10-16 17:49:37 -0700116 private final InternalLinkProvider listener = new InternalLinkProvider();
117
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800118 private final InternalRoleListener roleListener = new InternalRoleListener();
119
alshabib7911a052014-10-16 17:49:37 -0700120 protected final Map<DeviceId, LinkDiscovery> discoverers = new ConcurrentHashMap<>();
121
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800122 private SuppressionRules rules;
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -0800123 private ApplicationId appId;
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800124
alshabib7911a052014-10-16 17:49:37 -0700125 /**
126 * Creates an OpenFlow link provider.
127 */
128 public LLDPLinkProvider() {
Brian O'Connorabafb502014-12-02 22:26:20 -0800129 super(new ProviderId("lldp", "org.onosproject.provider.lldp"));
alshabib7911a052014-10-16 17:49:37 -0700130 }
131
132 @Activate
Saurav Dasc313c402015-02-27 10:09:47 -0800133 public void activate(ComponentContext context) {
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -0800134 appId =
135 coreService.registerApplication("org.onosproject.provider.lldp");
136
Saurav Dasc313c402015-02-27 10:09:47 -0800137 // to load configuration at startup
138 modified(context);
139 if (disableLD) {
140 log.info("Link Discovery has been permanently disabled by configuration");
141 return;
142 }
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800143
alshabib7911a052014-10-16 17:49:37 -0700144 providerService = providerRegistry.register(this);
145 deviceService.addListener(listener);
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -0800146 packetService.addProcessor(listener, 0);
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800147 masterService.addListener(roleListener);
148
alshabibdfc7afb2014-10-21 20:13:27 -0700149 LinkDiscovery ld;
alshabib5dc5a342014-12-03 14:11:16 -0800150 for (Device device : deviceService.getAvailableDevices()) {
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800151 if (rules.isSuppressed(device)) {
152 log.debug("LinkDiscovery from {} disabled by configuration", device.id());
153 continue;
154 }
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -0800155 ld = new LinkDiscovery(device, packetService, masterService,
alshabibdfc7afb2014-10-21 20:13:27 -0700156 providerService, useBDDP);
157 discoverers.put(device.id(), ld);
158 for (Port p : deviceService.getPorts(device.id())) {
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800159 if (rules.isSuppressed(p)) {
160 log.debug("LinkDiscovery from {}@{} disabled by configuration",
161 p.number(), device.id());
162 continue;
163 }
Yuta HIGUCHI00b476f2014-10-25 21:33:07 -0700164 if (!p.number().isLogical()) {
165 ld.addPort(p);
166 }
alshabibdfc7afb2014-10-21 20:13:27 -0700167 }
168 }
alshabib7911a052014-10-16 17:49:37 -0700169
Thomas Vachuska6f94ded2015-02-21 14:02:38 -0800170 executor = newSingleThreadScheduledExecutor(groupedThreads("onos/device", "sync-%d"));
171 executor.scheduleAtFixedRate(new SyncDeviceInfoTask(), INIT_DELAY, DELAY, SECONDS);
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800172
Jonathan Hart3cfce8e2015-01-14 16:43:27 -0800173 requestPackets();
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -0800174
alshabib7911a052014-10-16 17:49:37 -0700175 log.info("Started");
176 }
177
178 @Deactivate
179 public void deactivate() {
Saurav Dasc313c402015-02-27 10:09:47 -0800180 if (disableLD) {
181 return;
182 }
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800183 executor.shutdownNow();
alshabib7911a052014-10-16 17:49:37 -0700184 for (LinkDiscovery ld : discoverers.values()) {
185 ld.stop();
186 }
187 providerRegistry.unregister(this);
188 deviceService.removeListener(listener);
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -0800189 packetService.removeProcessor(listener);
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800190 masterService.removeListener(roleListener);
alshabib7911a052014-10-16 17:49:37 -0700191 providerService = null;
192
193 log.info("Stopped");
194 }
195
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800196 @Modified
197 public void modified(ComponentContext context) {
198 if (context == null) {
Saurav Dasc313c402015-02-27 10:09:47 -0800199 loadSuppressionRules();
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800200 return;
201 }
202 @SuppressWarnings("rawtypes")
203 Dictionary properties = context.getProperties();
204
Saurav Dasc313c402015-02-27 10:09:47 -0800205 String s = (String) properties.get(PROP_DISABLE_LD);
206 if (!Strings.isNullOrEmpty(s)) {
207 disableLD = Boolean.valueOf(s);
208 }
209 s = (String) properties.get(PROP_USE_BDDP);
210 if (!Strings.isNullOrEmpty(s)) {
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800211 useBDDP = Boolean.valueOf(s);
212 }
213 s = (String) properties.get(PROP_LLDP_SUPPRESSION);
Saurav Dasc313c402015-02-27 10:09:47 -0800214 if (!Strings.isNullOrEmpty(s)) {
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800215 filePath = s;
216 }
217
218 loadSuppressionRules();
219 }
220
221 private void loadSuppressionRules() {
222 SuppressionRulesStore store = new SuppressionRulesStore(filePath);
223 try {
Saurav Dasc313c402015-02-27 10:09:47 -0800224 log.info("Reading suppression rules from {}", filePath);
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800225 rules = store.read();
226 } catch (IOException e) {
227 log.info("Failed to load {}, using built-in rules", filePath);
228 // default rule to suppress ROADM to maintain compatibility
229 rules = new SuppressionRules(ImmutableSet.of(),
230 EnumSet.of(Device.Type.ROADM),
231 ImmutableMap.of());
232 }
233
234 // should refresh discoverers when we need dynamic reconfiguration
235 }
236
Jonathan Hart3cfce8e2015-01-14 16:43:27 -0800237 private void requestPackets() {
238 TrafficSelector.Builder lldpSelector = DefaultTrafficSelector.builder();
239 lldpSelector.matchEthType(Ethernet.TYPE_LLDP);
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -0800240 packetService.requestPackets(lldpSelector.build(),
241 PacketPriority.CONTROL, appId);
Jonathan Hart3cfce8e2015-01-14 16:43:27 -0800242
243 if (useBDDP) {
244 TrafficSelector.Builder bddpSelector = DefaultTrafficSelector.builder();
245 bddpSelector.matchEthType(Ethernet.TYPE_BSN);
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -0800246 packetService.requestPackets(bddpSelector.build(),
247 PacketPriority.CONTROL, appId);
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -0800248 }
249 }
250
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800251 private class InternalRoleListener implements MastershipListener {
252
253 @Override
254 public void event(MastershipEvent event) {
255
256 if (MastershipEvent.Type.BACKUPS_CHANGED.equals(event.type())) {
257 // only need new master events
258 return;
259 }
260
261 DeviceId deviceId = event.subject();
262 Device device = deviceService.getDevice(deviceId);
263 if (device == null) {
264 log.warn("Device {} doesn't exist, or isn't there yet", deviceId);
265 return;
266 }
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800267 if (rules.isSuppressed(device)) {
268 return;
269 }
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800270 synchronized (discoverers) {
271 if (!discoverers.containsKey(deviceId)) {
alshabib4785eec2014-12-04 16:45:45 -0800272 // ideally, should never reach here
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800273 log.debug("Device mastership changed ({}) {}",
274 event.type(), deviceId);
275 discoverers.put(deviceId, new LinkDiscovery(device,
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -0800276 packetService, masterService, providerService,
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800277 useBDDP));
278 }
279 }
280 }
281
282 }
alshabib7911a052014-10-16 17:49:37 -0700283
284 private class InternalLinkProvider implements PacketProcessor, DeviceListener {
285
286 @Override
287 public void event(DeviceEvent event) {
288 LinkDiscovery ld = null;
289 Device device = event.subject();
alshabibacd91832014-10-17 14:38:41 -0700290 Port port = event.port();
alshabibdfc7afb2014-10-21 20:13:27 -0700291 if (device == null) {
292 log.error("Device is null.");
293 return;
294 }
Yuta HIGUCHIeb24e9d2014-10-26 19:34:20 -0700295 log.trace("{} {} {}", event.type(), event.subject(), event);
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700296 final DeviceId deviceId = device.id();
alshabib7911a052014-10-16 17:49:37 -0700297 switch (event.type()) {
298 case DEVICE_ADDED:
Yuta HIGUCHIeb24e9d2014-10-26 19:34:20 -0700299 case DEVICE_UPDATED:
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800300 synchronized (discoverers) {
301 ld = discoverers.get(deviceId);
302 if (ld == null) {
303 if (rules.isSuppressed(device)) {
304 log.debug("LinkDiscovery from {} disabled by configuration", device.id());
305 return;
306 }
307 log.debug("Device added ({}) {}", event.type(),
308 deviceId);
309 discoverers.put(deviceId, new LinkDiscovery(device,
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -0800310 packetService, masterService, providerService, useBDDP));
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800311 } else {
312 if (ld.isStopped()) {
313 log.debug("Device restarted ({}) {}", event.type(),
314 deviceId);
315 ld.start();
316 }
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800317 }
Yuta HIGUCHIeb24e9d2014-10-26 19:34:20 -0700318 }
alshabib7911a052014-10-16 17:49:37 -0700319 break;
320 case PORT_ADDED:
321 case PORT_UPDATED:
Yuta HIGUCHIf6725882014-10-29 15:25:51 -0700322 if (port.isEnabled()) {
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700323 ld = discoverers.get(deviceId);
alshabib7911a052014-10-16 17:49:37 -0700324 if (ld == null) {
325 return;
326 }
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800327 if (rules.isSuppressed(port)) {
328 log.debug("LinkDiscovery from {}@{} disabled by configuration",
329 port.number(), device.id());
330 return;
331 }
Yuta HIGUCHI00b476f2014-10-25 21:33:07 -0700332 if (!port.number().isLogical()) {
Yuta HIGUCHIeb24e9d2014-10-26 19:34:20 -0700333 log.debug("Port added {}", port);
Yuta HIGUCHI00b476f2014-10-25 21:33:07 -0700334 ld.addPort(port);
335 }
alshabib7911a052014-10-16 17:49:37 -0700336 } else {
Yuta HIGUCHIf6725882014-10-29 15:25:51 -0700337 log.debug("Port down {}", port);
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700338 ConnectPoint point = new ConnectPoint(deviceId,
alshabibacd91832014-10-17 14:38:41 -0700339 port.number());
alshabib7911a052014-10-16 17:49:37 -0700340 providerService.linksVanished(point);
341 }
342 break;
343 case PORT_REMOVED:
Yuta HIGUCHIeb24e9d2014-10-26 19:34:20 -0700344 log.debug("Port removed {}", port);
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700345 ConnectPoint point = new ConnectPoint(deviceId,
alshabibacd91832014-10-17 14:38:41 -0700346 port.number());
alshabib7911a052014-10-16 17:49:37 -0700347 providerService.linksVanished(point);
alshabib4785eec2014-12-04 16:45:45 -0800348
alshabib7911a052014-10-16 17:49:37 -0700349 break;
350 case DEVICE_REMOVED:
351 case DEVICE_SUSPENDED:
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700352 log.debug("Device removed {}", deviceId);
353 ld = discoverers.get(deviceId);
alshabib7911a052014-10-16 17:49:37 -0700354 if (ld == null) {
355 return;
356 }
357 ld.stop();
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700358 providerService.linksVanished(deviceId);
alshabib7911a052014-10-16 17:49:37 -0700359 break;
360 case DEVICE_AVAILABILITY_CHANGED:
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700361 ld = discoverers.get(deviceId);
alshabib7911a052014-10-16 17:49:37 -0700362 if (ld == null) {
363 return;
364 }
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700365 if (deviceService.isAvailable(deviceId)) {
366 log.debug("Device up {}", deviceId);
alshabib7911a052014-10-16 17:49:37 -0700367 ld.start();
368 } else {
Yuta HIGUCHId19f6702014-10-31 15:23:25 -0700369 providerService.linksVanished(deviceId);
370 log.debug("Device down {}", deviceId);
alshabib7911a052014-10-16 17:49:37 -0700371 ld.stop();
372 }
373 break;
alshabib7911a052014-10-16 17:49:37 -0700374 default:
375 log.debug("Unknown event {}", event);
376 }
377 }
378
379 @Override
380 public void process(PacketContext context) {
alshabib4a179dc2014-10-17 17:17:01 -0700381 if (context == null) {
382 return;
383 }
alshabib7911a052014-10-16 17:49:37 -0700384 LinkDiscovery ld = discoverers.get(
385 context.inPacket().receivedFrom().deviceId());
386 if (ld == null) {
387 return;
388 }
389
390 if (ld.handleLLDP(context)) {
391 context.block();
392 }
393 }
394 }
395
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800396 private final class SyncDeviceInfoTask implements Runnable {
397
398 @Override
399 public void run() {
400 if (Thread.currentThread().isInterrupted()) {
401 log.info("Interrupted, quitting");
402 return;
403 }
404 // check what deviceService sees, to see if we are missing anything
405 try {
406 LinkDiscovery ld = null;
407 for (Device dev : deviceService.getDevices()) {
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800408 if (rules.isSuppressed(dev)) {
409 continue;
410 }
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800411 DeviceId did = dev.id();
412 synchronized (discoverers) {
413 if (!discoverers.containsKey(did)) {
Marc De Leenheer8b3e80b2015-03-06 14:27:03 -0800414 ld = new LinkDiscovery(dev, packetService,
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800415 masterService, providerService, useBDDP);
416 discoverers.put(did, ld);
417 for (Port p : deviceService.getPorts(did)) {
Yuta HIGUCHI41289382014-12-19 17:47:12 -0800418 if (rules.isSuppressed(p)) {
419 continue;
420 }
Ayaka Koshibeccfa94c2014-11-20 11:15:52 -0800421 if (!p.number().isLogical()) {
422 ld.addPort(p);
423 }
424 }
425 }
426 }
427 }
428 } catch (Exception e) {
429 // catch all Exception to avoid Scheduled task being suppressed.
430 log.error("Exception thrown during synchronization process", e);
431 }
432 }
433 }
434
alshabib7911a052014-10-16 17:49:37 -0700435}