blob: 4d97af79450215d91ef5176b3a6eef87f9a65f2f [file] [log] [blame]
Thomas Vachuska781d18b2014-10-27 10:31:25 -07001/*
Brian O'Connora09fe5b2017-08-03 21:12:30 -07002 * Copyright 2014-present Open Networking Foundation
Thomas Vachuska781d18b2014-10-27 10:31:25 -07003 *
Thomas Vachuska4f1a60c2014-10-28 13:39:07 -07004 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
Thomas Vachuska781d18b2014-10-27 10:31:25 -07007 *
Thomas Vachuska4f1a60c2014-10-28 13:39:07 -07008 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
Thomas Vachuska781d18b2014-10-27 10:31:25 -070015 */
Brian O'Connorabafb502014-12-02 22:26:20 -080016package org.onosproject.provider.host.impl;
alshabibe1cf87d2014-10-17 09:23:50 -070017
Charles Chan35a32322017-08-14 11:42:11 -070018import com.google.common.collect.Sets;
alshabibe1cf87d2014-10-17 09:23:50 -070019import org.apache.felix.scr.annotations.Activate;
20import org.apache.felix.scr.annotations.Component;
21import org.apache.felix.scr.annotations.Deactivate;
Thomas Vachuska33601602014-11-19 03:32:15 -080022import org.apache.felix.scr.annotations.Modified;
23import org.apache.felix.scr.annotations.Property;
alshabibe1cf87d2014-10-17 09:23:50 -070024import org.apache.felix.scr.annotations.Reference;
25import org.apache.felix.scr.annotations.ReferenceCardinality;
Jonathan Harte8600eb2015-01-12 10:30:45 -080026import org.onlab.packet.ARP;
Yi Tsengfcf5dce2017-07-26 14:30:41 -070027import org.onlab.packet.BasePacket;
Charles Chane6067892016-11-17 10:23:46 -080028import org.onlab.packet.DHCP;
Yi Tsengfcf5dce2017-07-26 14:30:41 -070029import org.onlab.packet.DHCP6;
Jonathan Harte8600eb2015-01-12 10:30:45 -080030import org.onlab.packet.Ethernet;
Pavlin Radoslavovd6612f92015-02-23 13:53:32 -080031import org.onlab.packet.ICMP6;
Thomas Vachuskaf845cf62015-03-24 10:13:09 -070032import org.onlab.packet.IPacket;
Charles Chane6067892016-11-17 10:23:46 -080033import org.onlab.packet.IPv4;
Jonathan Harte8600eb2015-01-12 10:30:45 -080034import org.onlab.packet.IPv6;
Charles Chan35a32322017-08-14 11:42:11 -070035import org.onlab.packet.Ip4Address;
Yi Tsengfcf5dce2017-07-26 14:30:41 -070036import org.onlab.packet.Ip6Address;
Jonathan Harte8600eb2015-01-12 10:30:45 -080037import org.onlab.packet.IpAddress;
Charles M.C. Chan956cb692015-04-26 18:49:39 +080038import org.onlab.packet.MacAddress;
Charles Chane6067892016-11-17 10:23:46 -080039import org.onlab.packet.UDP;
Jonathan Harte8600eb2015-01-12 10:30:45 -080040import org.onlab.packet.VlanId;
Yi Tsengfcf5dce2017-07-26 14:30:41 -070041import org.onlab.packet.dhcp.Dhcp6ClientIdOption;
42import org.onlab.packet.dhcp.Dhcp6IaAddressOption;
43import org.onlab.packet.dhcp.Dhcp6IaNaOption;
44import org.onlab.packet.dhcp.Dhcp6IaTaOption;
45import org.onlab.packet.dhcp.Dhcp6RelayOption;
Charles M.C. Chan956cb692015-04-26 18:49:39 +080046import org.onlab.packet.ipv6.IExtensionHeader;
Jonathan Harte8600eb2015-01-12 10:30:45 -080047import org.onlab.packet.ndp.NeighborAdvertisement;
48import org.onlab.packet.ndp.NeighborSolicitation;
Charles M.C. Chan441d7da2015-03-17 21:03:39 +080049import org.onlab.packet.ndp.RouterAdvertisement;
50import org.onlab.packet.ndp.RouterSolicitation;
Jian Lid9b5f552016-03-11 18:15:31 -080051import org.onlab.util.Tools;
Thomas Vachuska6519e6f2015-03-11 02:29:31 -070052import org.onosproject.cfg.ComponentConfigService;
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -080053import org.onosproject.core.ApplicationId;
54import org.onosproject.core.CoreService;
Ray Milkeyfacf2862017-08-03 11:58:29 -070055import org.onosproject.net.intf.InterfaceService;
Brian O'Connorabafb502014-12-02 22:26:20 -080056import org.onosproject.net.ConnectPoint;
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -080057import org.onosproject.net.Device;
Charles Chan888e20a2017-05-01 15:44:23 -070058import org.onosproject.net.DeviceId;
Brian O'Connorabafb502014-12-02 22:26:20 -080059import org.onosproject.net.Host;
60import org.onosproject.net.HostId;
61import org.onosproject.net.HostLocation;
62import org.onosproject.net.device.DeviceEvent;
63import org.onosproject.net.device.DeviceListener;
64import org.onosproject.net.device.DeviceService;
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -080065import org.onosproject.net.flow.DefaultTrafficSelector;
sdn94b00152016-08-30 02:12:32 -070066import org.onosproject.net.flow.DefaultTrafficTreatment;
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -080067import org.onosproject.net.flow.TrafficSelector;
sdn94b00152016-08-30 02:12:32 -070068import org.onosproject.net.flow.TrafficTreatment;
Brian O'Connorabafb502014-12-02 22:26:20 -080069import org.onosproject.net.host.DefaultHostDescription;
70import org.onosproject.net.host.HostDescription;
71import org.onosproject.net.host.HostProvider;
72import org.onosproject.net.host.HostProviderRegistry;
73import org.onosproject.net.host.HostProviderService;
74import org.onosproject.net.host.HostService;
sdn94b00152016-08-30 02:12:32 -070075import org.onosproject.net.packet.DefaultOutboundPacket;
76import org.onosproject.net.packet.OutboundPacket;
Brian O'Connorabafb502014-12-02 22:26:20 -080077import org.onosproject.net.packet.PacketContext;
Jonathan Hart3cfce8e2015-01-14 16:43:27 -080078import org.onosproject.net.packet.PacketPriority;
Brian O'Connorabafb502014-12-02 22:26:20 -080079import org.onosproject.net.packet.PacketProcessor;
80import org.onosproject.net.packet.PacketService;
81import org.onosproject.net.provider.AbstractProvider;
82import org.onosproject.net.provider.ProviderId;
83import org.onosproject.net.topology.Topology;
84import org.onosproject.net.topology.TopologyService;
Thomas Vachuska33601602014-11-19 03:32:15 -080085import org.osgi.service.component.ComponentContext;
alshabibe1cf87d2014-10-17 09:23:50 -070086import org.slf4j.Logger;
87
sdn94b00152016-08-30 02:12:32 -070088import java.nio.ByteBuffer;
Thomas Vachuskaf845cf62015-03-24 10:13:09 -070089import java.util.Dictionary;
Yi Tseng7a38f9a2017-06-09 14:36:40 -070090import java.util.Objects;
91import java.util.Optional;
Madan Jampania3770c32015-12-11 12:07:41 -080092import java.util.concurrent.ExecutorService;
Yi Tseng7a38f9a2017-06-09 14:36:40 -070093import java.util.stream.Stream;
Charles Chan35a32322017-08-14 11:42:11 -070094import java.util.Set;
Thomas Vachuskaf845cf62015-03-24 10:13:09 -070095
Madan Jampania3770c32015-12-11 12:07:41 -080096import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
97import static org.onlab.util.Tools.groupedThreads;
Thomas Vachuskaf845cf62015-03-24 10:13:09 -070098import static org.slf4j.LoggerFactory.getLogger;
99
alshabibe1cf87d2014-10-17 09:23:50 -0700100/**
Thomas Vachuskaec9c7dd2015-09-03 18:30:04 -0700101 * Provider which uses an OpenFlow controller to detect network end-station
102 * hosts.
alshabibe1cf87d2014-10-17 09:23:50 -0700103 */
104@Component(immediate = true)
105public class HostLocationProvider extends AbstractProvider implements HostProvider {
alshabibe1cf87d2014-10-17 09:23:50 -0700106 private final Logger log = getLogger(getClass());
107
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -0800108 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
109 protected CoreService coreService;
110
111 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
alshabibe1cf87d2014-10-17 09:23:50 -0700112 protected HostProviderRegistry providerRegistry;
113
114 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
Pavlin Radoslavovd6612f92015-02-23 13:53:32 -0800115 protected PacketService packetService;
alshabibe1cf87d2014-10-17 09:23:50 -0700116
117 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
118 protected TopologyService topologyService;
119
Thomas Vachuska33601602014-11-19 03:32:15 -0800120 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
121 protected HostService hostService;
122
123 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
124 protected DeviceService deviceService;
125
Thomas Vachuska6519e6f2015-03-11 02:29:31 -0700126 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
127 protected ComponentConfigService cfgService;
128
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700129 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
130 protected InterfaceService interfaceService;
131
alshabibe1cf87d2014-10-17 09:23:50 -0700132 private HostProviderService providerService;
133
134 private final InternalHostProvider processor = new InternalHostProvider();
Thomas Vachuska33601602014-11-19 03:32:15 -0800135 private final DeviceListener deviceListener = new InternalDeviceListener();
136
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -0800137 private ApplicationId appId;
138
Thomas Vachuska33601602014-11-19 03:32:15 -0800139 @Property(name = "hostRemovalEnabled", boolValue = true,
140 label = "Enable host removal on port/device down events")
141 private boolean hostRemovalEnabled = true;
alshabibe1cf87d2014-10-17 09:23:50 -0700142
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700143 @Property(name = "requestArp", boolValue = true,
144 label = "Request ARP packets for neighbor discovery by the " +
Charles Chane6067892016-11-17 10:23:46 -0800145 "Host Location Provider; default is true")
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700146 private boolean requestArp = true;
Charles Chane6067892016-11-17 10:23:46 -0800147
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700148 @Property(name = "requestIpv6ND", boolValue = false,
149 label = "Requests IPv6 Neighbor Discovery by the " +
Thomas Vachuska27bee092015-06-23 19:03:10 -0700150 "Host Location Provider; default is false")
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700151 private boolean requestIpv6ND = false;
Charles Chane6067892016-11-17 10:23:46 -0800152
153 @Property(name = "useDhcp", boolValue = false,
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700154 label = "Use DHCP for neighbor discovery by the " +
Charles Chane6067892016-11-17 10:23:46 -0800155 "Host Location Provider; default is false")
156 private boolean useDhcp = false;
alshabibe1cf87d2014-10-17 09:23:50 -0700157
Yi Tsengfcf5dce2017-07-26 14:30:41 -0700158 @Property(name = "useDhcp6", boolValue = false,
159 label = "Use DHCPv6 for neighbor discovery by the " +
160 "Host Location Provider; default is false")
161 private boolean useDhcp6 = false;
162
Jonathan Hart9af322d2016-01-06 17:42:04 -0800163 @Property(name = "requestInterceptsEnabled", boolValue = true,
164 label = "Enable requesting packet intercepts")
165 private boolean requestInterceptsEnabled = true;
166
Charles Chan35a32322017-08-14 11:42:11 -0700167 @Property(name = "multihomingEnabled", boolValue = false,
168 label = "Allow hosts to be multihomed")
169 private boolean multihomingEnabled = false;
Madan Jampania3770c32015-12-11 12:07:41 -0800170
Charles Chan35a32322017-08-14 11:42:11 -0700171 protected ExecutorService eventHandler;
sdn94b00152016-08-30 02:12:32 -0700172
alshabibe1cf87d2014-10-17 09:23:50 -0700173 /**
174 * Creates an OpenFlow host provider.
175 */
176 public HostLocationProvider() {
Brian O'Connorabafb502014-12-02 22:26:20 -0800177 super(new ProviderId("of", "org.onosproject.provider.host"));
alshabibe1cf87d2014-10-17 09:23:50 -0700178 }
179
180 @Activate
Thomas Vachuska33601602014-11-19 03:32:15 -0800181 public void activate(ComponentContext context) {
Thomas Vachuska6519e6f2015-03-11 02:29:31 -0700182 cfgService.registerProperties(getClass());
183 appId = coreService.registerApplication("org.onosproject.provider.host");
Jonathan Hart9af322d2016-01-06 17:42:04 -0800184 eventHandler = newSingleThreadScheduledExecutor(
HIGUCHI Yutad9e01052016-04-14 09:31:42 -0700185 groupedThreads("onos/host-loc-provider", "event-handler", log));
alshabibe1cf87d2014-10-17 09:23:50 -0700186 providerService = providerRegistry.register(this);
Brian O'Connor3b783262015-07-29 17:49:24 -0700187 packetService.addProcessor(processor, PacketProcessor.advisor(1));
Thomas Vachuska33601602014-11-19 03:32:15 -0800188 deviceService.addListener(deviceListener);
Jonathan Hart9af322d2016-01-06 17:42:04 -0800189
190 modified(context);
Jonathan Hart3cfce8e2015-01-14 16:43:27 -0800191
Charles M.C. Chane148de82015-05-06 12:38:21 +0800192 log.info("Started with Application ID {}", appId.id());
193 }
194
195 @Deactivate
196 public void deactivate() {
Charles M.C. Chane148de82015-05-06 12:38:21 +0800197 cfgService.unregisterProperties(getClass(), false);
Thomas Vachuska27bee092015-06-23 19:03:10 -0700198
199 withdrawIntercepts();
200
Charles M.C. Chane148de82015-05-06 12:38:21 +0800201 providerRegistry.unregister(this);
202 packetService.removeProcessor(processor);
203 deviceService.removeListener(deviceListener);
Madan Jampania3770c32015-12-11 12:07:41 -0800204 eventHandler.shutdown();
Charles M.C. Chane148de82015-05-06 12:38:21 +0800205 providerService = null;
206 log.info("Stopped");
207 }
208
209 @Modified
210 public void modified(ComponentContext context) {
Charles M.C. Chane148de82015-05-06 12:38:21 +0800211 readComponentConfiguration(context);
Jonathan Hart9af322d2016-01-06 17:42:04 -0800212
213 if (requestInterceptsEnabled) {
214 requestIntercepts();
215 } else {
216 withdrawIntercepts();
217 }
Charles M.C. Chane148de82015-05-06 12:38:21 +0800218 }
219
220 /**
Thomas Vachuska27bee092015-06-23 19:03:10 -0700221 * Request packet intercepts.
Charles M.C. Chane148de82015-05-06 12:38:21 +0800222 */
Thomas Vachuska27bee092015-06-23 19:03:10 -0700223 private void requestIntercepts() {
Charles Chane6067892016-11-17 10:23:46 -0800224 // Use ARP
225 TrafficSelector arpSelector = DefaultTrafficSelector.builder()
226 .matchEthType(Ethernet.TYPE_ARP)
227 .build();
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700228 if (requestArp) {
Charles Chane6067892016-11-17 10:23:46 -0800229 packetService.requestPackets(arpSelector, PacketPriority.CONTROL, appId);
Thomas Vachuska27bee092015-06-23 19:03:10 -0700230 } else {
Charles Chane6067892016-11-17 10:23:46 -0800231 packetService.cancelPackets(arpSelector, PacketPriority.CONTROL, appId);
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800232 }
Thomas Vachuska27bee092015-06-23 19:03:10 -0700233
Charles Chane6067892016-11-17 10:23:46 -0800234 // Use IPv6 Neighbor Discovery
235 TrafficSelector ipv6NsSelector = DefaultTrafficSelector.builder()
236 .matchEthType(Ethernet.TYPE_IPV6)
237 .matchIPProtocol(IPv6.PROTOCOL_ICMP6)
238 .matchIcmpv6Type(ICMP6.NEIGHBOR_SOLICITATION)
239 .build();
240 TrafficSelector ipv6NaSelector = DefaultTrafficSelector.builder()
241 .matchEthType(Ethernet.TYPE_IPV6)
242 .matchIPProtocol(IPv6.PROTOCOL_ICMP6)
243 .matchIcmpv6Type(ICMP6.NEIGHBOR_ADVERTISEMENT)
244 .build();
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700245 if (requestIpv6ND) {
Charles Chane6067892016-11-17 10:23:46 -0800246 packetService.requestPackets(ipv6NsSelector, PacketPriority.CONTROL, appId);
247 packetService.requestPackets(ipv6NaSelector, PacketPriority.CONTROL, appId);
Thomas Vachuska27bee092015-06-23 19:03:10 -0700248 } else {
Charles Chane6067892016-11-17 10:23:46 -0800249 packetService.cancelPackets(ipv6NsSelector, PacketPriority.CONTROL, appId);
250 packetService.cancelPackets(ipv6NaSelector, PacketPriority.CONTROL, appId);
251 }
Thomas Vachuska27bee092015-06-23 19:03:10 -0700252 }
253
254 /**
255 * Withdraw packet intercepts.
256 */
257 private void withdrawIntercepts() {
258 TrafficSelector.Builder selector = DefaultTrafficSelector.builder();
259 selector.matchEthType(Ethernet.TYPE_ARP);
Aaron Kruglikov07a923d2015-07-03 13:30:57 -0700260 packetService.cancelPackets(selector.build(), PacketPriority.CONTROL, appId);
Thomas Vachuska27bee092015-06-23 19:03:10 -0700261
262 // IPv6 Neighbor Solicitation packet.
263 selector.matchEthType(Ethernet.TYPE_IPV6);
264 selector.matchIPProtocol(IPv6.PROTOCOL_ICMP6);
265 selector.matchIcmpv6Type(ICMP6.NEIGHBOR_SOLICITATION);
266 packetService.cancelPackets(selector.build(), PacketPriority.CONTROL, appId);
267
268 // IPv6 Neighbor Advertisement packet.
269 selector.matchIcmpv6Type(ICMP6.NEIGHBOR_ADVERTISEMENT);
270 packetService.cancelPackets(selector.build(), PacketPriority.CONTROL, appId);
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800271 }
272
273 /**
274 * Extracts properties from the component configuration context.
275 *
276 * @param context the component context
277 */
278 private void readComponentConfiguration(ComponentContext context) {
279 Dictionary<?, ?> properties = context.getProperties();
280 Boolean flag;
281
Jian Lid9b5f552016-03-11 18:15:31 -0800282 flag = Tools.isPropertyEnabled(properties, "hostRemovalEnabled");
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800283 if (flag == null) {
284 log.info("Host removal on port/device down events is not configured, " +
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700285 "using current value of {}", hostRemovalEnabled);
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800286 } else {
287 hostRemovalEnabled = flag;
288 log.info("Configured. Host removal on port/device down events is {}",
289 hostRemovalEnabled ? "enabled" : "disabled");
Thomas Vachuska33601602014-11-19 03:32:15 -0800290 }
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800291
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700292 flag = Tools.isPropertyEnabled(properties, "requestArp");
Charles Chane6067892016-11-17 10:23:46 -0800293 if (flag == null) {
294 log.info("Using ARP is not configured, " +
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700295 "using current value of {}", requestArp);
Charles Chane6067892016-11-17 10:23:46 -0800296 } else {
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700297 requestArp = flag;
Charles Chane6067892016-11-17 10:23:46 -0800298 log.info("Configured. Using ARP is {}",
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700299 requestArp ? "enabled" : "disabled");
Charles Chane6067892016-11-17 10:23:46 -0800300 }
301
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700302 flag = Tools.isPropertyEnabled(properties, "requestIpv6ND");
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800303 if (flag == null) {
304 log.info("Using IPv6 Neighbor Discovery is not configured, " +
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700305 "using current value of {}", requestIpv6ND);
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800306 } else {
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700307 requestIpv6ND = flag;
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800308 log.info("Configured. Using IPv6 Neighbor Discovery is {}",
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700309 requestIpv6ND ? "enabled" : "disabled");
Charles Chane6067892016-11-17 10:23:46 -0800310 }
311
312 flag = Tools.isPropertyEnabled(properties, "useDhcp");
313 if (flag == null) {
314 log.info("Using DHCP is not configured, " +
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700315 "using current value of {}", useDhcp);
Charles Chane6067892016-11-17 10:23:46 -0800316 } else {
317 useDhcp = flag;
318 log.info("Configured. Using DHCP is {}",
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700319 useDhcp ? "enabled" : "disabled");
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800320 }
Jonathan Hart9af322d2016-01-06 17:42:04 -0800321
Jian Lid9b5f552016-03-11 18:15:31 -0800322 flag = Tools.isPropertyEnabled(properties, "requestInterceptsEnabled");
Jonathan Hart9af322d2016-01-06 17:42:04 -0800323 if (flag == null) {
324 log.info("Request intercepts is not configured, " +
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700325 "using current value of {}", requestInterceptsEnabled);
Jonathan Hart9af322d2016-01-06 17:42:04 -0800326 } else {
327 requestInterceptsEnabled = flag;
328 log.info("Configured. Request intercepts is {}",
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700329 requestInterceptsEnabled ? "enabled" : "disabled");
Jonathan Hart9af322d2016-01-06 17:42:04 -0800330 }
Charles Chan35a32322017-08-14 11:42:11 -0700331
332 flag = Tools.isPropertyEnabled(properties, "multihomingEnabled");
333 if (flag == null) {
334 log.info("Multihoming is not configured, " +
335 "using current value of {}", multihomingEnabled);
336 } else {
337 multihomingEnabled = flag;
338 log.info("Configured. Multihoming is {}",
339 multihomingEnabled ? "enabled" : "disabled");
340 }
341
Pavlin Radoslavov93b606b2015-02-25 17:28:39 -0800342 }
343
alshabibe1cf87d2014-10-17 09:23:50 -0700344 @Override
345 public void triggerProbe(Host host) {
Saurav Dasb0ae6ee2017-03-04 16:08:47 -0800346 //log.info("Triggering probe on device {} ", host);
Jonathan Hartf353e402016-09-23 09:27:36 -0700347
348 // FIXME Disabling host probing for now, because sending packets from a
349 // broadcast MAC address caused problems when two ONOS networks were
350 // interconnected. Host probing should take into account the interface
351 // configuration when determining which source address to use.
352
353 //MastershipRole role = deviceService.getRole(host.location().deviceId());
354 //if (role.equals(MastershipRole.MASTER)) {
355 // host.ipAddresses().forEach(ip -> {
356 // sendProbe(host, ip);
357 // });
358 //} else {
359 // log.info("not the master, master will probe {}");
360 //}
sdn94b00152016-08-30 02:12:32 -0700361 }
362
363 private void sendProbe(Host host, IpAddress targetIp) {
364 Ethernet probePacket = null;
365 if (targetIp.isIp4()) {
366 // IPv4: Use ARP
367 probePacket = buildArpRequest(targetIp, host);
368 } else {
369 // IPv6: Use Neighbor Discovery
Jonathan Hartf353e402016-09-23 09:27:36 -0700370 //TODO need to implement ndp probe
sdn94b00152016-08-30 02:12:32 -0700371 log.info("Triggering probe on device {} ", host);
372 }
373
374 TrafficTreatment treatment = DefaultTrafficTreatment.builder().setOutput(host.location().port()).build();
375
376 OutboundPacket outboundPacket = new DefaultOutboundPacket(host.location().deviceId(), treatment,
377 ByteBuffer.wrap(probePacket.serialize()));
378
379 packetService.emit(outboundPacket);
380 }
381
Charles Chan888e20a2017-05-01 15:44:23 -0700382 // This method is using source ip as 0.0.0.0 , to receive the reply even from the sub net hosts.
sdn94b00152016-08-30 02:12:32 -0700383 private Ethernet buildArpRequest(IpAddress targetIp, Host host) {
Charles Chan35a32322017-08-14 11:42:11 -0700384 return ARP.buildArpRequest(MacAddress.BROADCAST.toBytes(), Ip4Address.ZERO.toOctets(),
385 MacAddress.BROADCAST.toBytes(), targetIp.toOctets(),
386 MacAddress.BROADCAST.toBytes(), VlanId.NONE.toShort());
alshabibe1cf87d2014-10-17 09:23:50 -0700387 }
388
389 private class InternalHostProvider implements PacketProcessor {
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800390 /**
Charles Chan888e20a2017-05-01 15:44:23 -0700391 * Create or update host information.
392 * Will not update IP if IP is null, all zero or self-assigned.
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800393 *
Thomas Vachuska27bee092015-06-23 19:03:10 -0700394 * @param hid host ID
395 * @param mac source Mac address
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800396 * @param vlan VLAN ID
397 * @param hloc host location
Charles Chan888e20a2017-05-01 15:44:23 -0700398 * @param ip source IP address or null if not updating
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800399 */
Charles Chan888e20a2017-05-01 15:44:23 -0700400 private void createOrUpdateHost(HostId hid, MacAddress mac,
401 VlanId vlan, HostLocation hloc,
402 IpAddress ip) {
Charles Chan35a32322017-08-14 11:42:11 -0700403 Set<HostLocation> newLocations = Sets.newHashSet(hloc);
404
405 if (multihomingEnabled) {
406 Host existingHost = hostService.getHost(hid);
407 if (existingHost != null) {
408 Set<HostLocation> prevLocations = existingHost.locations();
409 newLocations.addAll(prevLocations);
410
411 if (!existingHost.locations().contains(hloc)) {
412 probeLocations(existingHost);
413 }
414 }
415 }
416
Charles Chan888e20a2017-05-01 15:44:23 -0700417 HostDescription desc = ip == null || ip.isZero() || ip.isSelfAssigned() ?
Charles Chan35a32322017-08-14 11:42:11 -0700418 new DefaultHostDescription(mac, vlan, newLocations, Sets.newHashSet(), false) :
419 new DefaultHostDescription(mac, vlan, newLocations, Sets.newHashSet(ip), false);
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700420 try {
Ray Milkeydc083442016-02-22 11:27:57 -0800421 providerService.hostDetected(hid, desc, false);
Sahil Lele3a0cdd52015-07-21 14:16:31 -0700422 } catch (IllegalStateException e) {
423 log.debug("Host {} suppressed", hid);
424 }
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800425 }
alshabibe1cf87d2014-10-17 09:23:50 -0700426
Charles Chane6067892016-11-17 10:23:46 -0800427 /**
Charles Chan35a32322017-08-14 11:42:11 -0700428 * Start verification procedure of all previous locations by sending probes.
429 *
430 * @param host Host to be probed
431 */
432 private void probeLocations(Host host) {
433 host.locations().forEach(location -> {
434 MacAddress probeMac = providerService.addPendingHostLocation(host.id(), location);
435
436 host.ipAddresses().stream().findFirst().ifPresent(ip -> {
437 Ethernet probe;
438 if (ip.isIp4()) {
439 probe = ARP.buildArpRequest(probeMac.toBytes(), Ip4Address.ZERO.toOctets(),
440 host.id().mac().toBytes(), ip.toOctets(),
441 host.id().mac().toBytes(), host.id().vlanId().toShort());
442 } else {
443 probe = NeighborSolicitation.buildNdpSolicit(
444 ip.getIp6Address().toOctets(),
445 IPv6.getLinkLocalAddress(probeMac.toBytes()),
446 IPv6.getSolicitNodeAddress(ip.getIp6Address().toOctets()),
447 probeMac.toBytes(),
448 IPv6.getMCastMacAddress(ip.getIp6Address().toOctets()),
449 host.id().vlanId());
450 }
451 sendProbe(probe, location);
452 });
453 });
454 }
455
456 /**
457 * Send the probe packet on given port.
458 *
459 * @param probe the probe packet
460 * @param connectPoint the port we want to probe
461 */
462 private void sendProbe(Ethernet probe, ConnectPoint connectPoint) {
463 log.info("Probing host {} on location {} with probeMac {}",
464 probe.getDestinationMAC(), connectPoint, probe.getSourceMAC());
465 TrafficTreatment treatment = DefaultTrafficTreatment.builder().setOutput(connectPoint.port()).build();
466 OutboundPacket outboundPacket = new DefaultOutboundPacket(connectPoint.deviceId(),
467 treatment, ByteBuffer.wrap(probe.serialize()));
468 packetService.emit(outboundPacket);
469 }
470
471 /**
Charles Chan888e20a2017-05-01 15:44:23 -0700472 * Updates IP address for an existing host.
Charles Chane6067892016-11-17 10:23:46 -0800473 *
474 * @param hid host ID
475 * @param ip IP address
476 */
Charles Chan888e20a2017-05-01 15:44:23 -0700477 private void updateHostIp(HostId hid, IpAddress ip) {
Charles Chane6067892016-11-17 10:23:46 -0800478 Host host = hostService.getHost(hid);
479 if (host == null) {
Charles Chan35a32322017-08-14 11:42:11 -0700480 log.warn("Fail to update IP for {}. Host does not exist", hid);
Charles Chane6067892016-11-17 10:23:46 -0800481 return;
482 }
483
Charles Chan888e20a2017-05-01 15:44:23 -0700484 HostDescription desc = new DefaultHostDescription(hid.mac(), hid.vlanId(),
Charles Chan35a32322017-08-14 11:42:11 -0700485 host.locations(), Sets.newHashSet(ip), false);
Charles Chane6067892016-11-17 10:23:46 -0800486 try {
487 providerService.hostDetected(hid, desc, false);
488 } catch (IllegalStateException e) {
489 log.debug("Host {} suppressed", hid);
490 }
491 }
492
alshabibe1cf87d2014-10-17 09:23:50 -0700493 @Override
494 public void process(PacketContext context) {
alshabib4a179dc2014-10-17 17:17:01 -0700495 if (context == null) {
496 return;
497 }
alshabibe1cf87d2014-10-17 09:23:50 -0700498
Thomas Vachuskaf845cf62015-03-24 10:13:09 -0700499 Ethernet eth = context.inPacket().parsed();
Jonathan Harte8600eb2015-01-12 10:30:45 -0800500 if (eth == null) {
501 return;
502 }
Jonathan Hartf353e402016-09-23 09:27:36 -0700503
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800504 MacAddress srcMac = eth.getSourceMAC();
Jonathan Hartf353e402016-09-23 09:27:36 -0700505 if (srcMac.isBroadcast() || srcMac.isMulticast()) {
506 return;
507 }
Jonathan Harte8600eb2015-01-12 10:30:45 -0800508
alshabibe1cf87d2014-10-17 09:23:50 -0700509 VlanId vlan = VlanId.vlanId(eth.getVlanID());
510 ConnectPoint heardOn = context.inPacket().receivedFrom();
511
Thomas Vachuskaf845cf62015-03-24 10:13:09 -0700512 // If this arrived on control port, bail out.
513 if (heardOn.port().isLogical()) {
514 return;
515 }
516
alshabibe1cf87d2014-10-17 09:23:50 -0700517 // If this is not an edge port, bail out.
518 Topology topology = topologyService.currentTopology();
519 if (topologyService.isInfrastructure(topology, heardOn)) {
520 return;
521 }
522
Thomas Vachuskaec9c7dd2015-09-03 18:30:04 -0700523 HostLocation hloc = new HostLocation(heardOn, System.currentTimeMillis());
alshabibe1cf87d2014-10-17 09:23:50 -0700524 HostId hid = HostId.hostId(eth.getSourceMAC(), vlan);
Charles Chan35a32322017-08-14 11:42:11 -0700525 MacAddress destMac = eth.getDestinationMAC();
526
527 // Receives a location probe. Invalid entry from the cache
528 if (multihomingEnabled && destMac.isOnos() && !MacAddress.NONE.equals(destMac)) {
529 log.info("Receives probe for {}/{} on {}", srcMac, vlan, heardOn);
530 providerService.removePendingHostLocation(destMac);
531 return;
532 }
alshabibe1cf87d2014-10-17 09:23:50 -0700533
Charles M.C. Chan7fee36a2014-12-31 00:19:59 +0800534 // ARP: possible new hosts, update both location and IP
alshabibe1cf87d2014-10-17 09:23:50 -0700535 if (eth.getEtherType() == Ethernet.TYPE_ARP) {
536 ARP arp = (ARP) eth.getPayload();
Pavlin Radoslavovd6612f92015-02-23 13:53:32 -0800537 IpAddress ip = IpAddress.valueOf(IpAddress.Version.INET,
538 arp.getSenderProtocolAddress());
Charles Chan888e20a2017-05-01 15:44:23 -0700539 createOrUpdateHost(hid, srcMac, vlan, hloc, ip);
alshabibe1cf87d2014-10-17 09:23:50 -0700540
Charles Chan5d5e0622015-09-25 13:00:06 -0700541 // IPv4: update location only
alshabibe1cf87d2014-10-17 09:23:50 -0700542 } else if (eth.getEtherType() == Ethernet.TYPE_IPV4) {
Yi Tsengcfa637a2017-07-26 17:24:10 -0700543 DHCP dhcp = findDhcp(eth).orElse(null);
544 if (dhcp != null) {
545 if (useDhcp) {
546 // learn host (server or client) MAC address
547 createOrUpdateHost(hid, srcMac, vlan, hloc, null);
548
549 // DHCP ACK: additionally update IP of DHCP client
550 if (dhcp.getPacketType().equals(DHCP.MsgType.DHCPACK)) {
551 MacAddress hostMac = MacAddress.valueOf(dhcp.getClientHardwareAddress());
552 VlanId hostVlan = VlanId.vlanId(eth.getVlanID());
553 HostId hostId = HostId.hostId(hostMac, hostVlan);
554 updateHostIp(hostId, IpAddress.valueOf(dhcp.getYourIPAddress()));
555 }
556 }
557 } else {
558 // learn host MAC address
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700559 createOrUpdateHost(hid, srcMac, vlan, hloc, null);
Charles Chane6067892016-11-17 10:23:46 -0800560 }
Charles Chan5d5e0622015-09-25 13:00:06 -0700561 //
562 // NeighborAdvertisement and NeighborSolicitation: possible
563 // new hosts, update both location and IP.
564 //
565 // IPv6: update location only
Charles M.C. Chan7fee36a2014-12-31 00:19:59 +0800566 } else if (eth.getEtherType() == Ethernet.TYPE_IPV6) {
Charles M.C. Chan7fee36a2014-12-31 00:19:59 +0800567 IPv6 ipv6 = (IPv6) eth.getPayload();
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800568 IpAddress ip = IpAddress.valueOf(IpAddress.Version.INET6,
Thomas Vachuska27bee092015-06-23 19:03:10 -0700569 ipv6.getSourceAddress());
Charles M.C. Chan7fee36a2014-12-31 00:19:59 +0800570
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800571 // skip extension headers
572 IPacket pkt = ipv6;
573 while (pkt.getPayload() != null &&
574 pkt.getPayload() instanceof IExtensionHeader) {
575 pkt = pkt.getPayload();
576 }
Charles M.C. Chan9148d2d2015-04-27 03:36:39 +0800577 pkt = pkt.getPayload();
Yi Tsengfcf5dce2017-07-26 14:30:41 -0700578
579 // DHCPv6 protocol
580 DHCP6 dhcp6 = findDhcp6(pkt).orElse(null);
581 if (dhcp6 != null && useDhcp6) {
582 createOrUpdateHost(hid, srcMac, vlan, hloc, null);
583 handleDhcp6(dhcp6, vlan);
584 return;
585 }
586
Charles M.C. Chan9148d2d2015-04-27 03:36:39 +0800587 if (pkt != null && pkt instanceof ICMP6) {
Yi Tsengfcf5dce2017-07-26 14:30:41 -0700588 // Neighbor Discovery Protocol
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800589 pkt = pkt.getPayload();
590 // RouterSolicitation, RouterAdvertisement
Charles M.C. Chan9148d2d2015-04-27 03:36:39 +0800591 if (pkt != null && (pkt instanceof RouterAdvertisement ||
Thomas Vachuska27bee092015-06-23 19:03:10 -0700592 pkt instanceof RouterSolicitation)) {
Charles M.C. Chan441d7da2015-03-17 21:03:39 +0800593 return;
594 }
Charles M.C. Chan9148d2d2015-04-27 03:36:39 +0800595 if (pkt != null && (pkt instanceof NeighborSolicitation ||
Thomas Vachuska27bee092015-06-23 19:03:10 -0700596 pkt instanceof NeighborAdvertisement)) {
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800597 // Duplicate Address Detection
598 if (ip.isZero()) {
Charles M.C. Chan441d7da2015-03-17 21:03:39 +0800599 return;
Charles M.C. Chan7fee36a2014-12-31 00:19:59 +0800600 }
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800601 // NeighborSolicitation, NeighborAdvertisement
Charles Chan888e20a2017-05-01 15:44:23 -0700602 createOrUpdateHost(hid, srcMac, vlan, hloc, ip);
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800603 return;
Charles M.C. Chan7fee36a2014-12-31 00:19:59 +0800604 }
Charles M.C. Chan7fee36a2014-12-31 00:19:59 +0800605 }
Charles M.C. Chan956cb692015-04-26 18:49:39 +0800606
607 // multicast
608 if (eth.isMulticast()) {
609 return;
610 }
611
612 // normal IPv6 packets
Charles Chan888e20a2017-05-01 15:44:23 -0700613 createOrUpdateHost(hid, srcMac, vlan, hloc, null);
alshabibe1cf87d2014-10-17 09:23:50 -0700614 }
615 }
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700616
Yi Tsengfcf5dce2017-07-26 14:30:41 -0700617 /**
618 * Handles DHCPv6 packet, if message type is ACK, update IP address
619 * according to DHCPv6 payload (IA Address option).
620 *
621 * @param dhcp6 the DHCPv6 payload
622 * @param vlanId the vlan of this packet
623 */
624 private void handleDhcp6(DHCP6 dhcp6, VlanId vlanId) {
625 // extract the relay message if exist
626 while (dhcp6 != null && DHCP6.RELAY_MSG_TYPES.contains(dhcp6.getMsgType())) {
627 dhcp6 = dhcp6.getOptions().stream()
628 .filter(opt -> opt instanceof Dhcp6RelayOption)
629 .map(BasePacket::getPayload)
630 .map(pld -> (DHCP6) pld)
631 .findFirst()
632 .orElse(null);
633 }
634
635 if (dhcp6 == null) {
636 // Can't find dhcp payload
637 log.warn("Can't find dhcp payload from relay message");
638 return;
639 }
640
641 if (dhcp6.getMsgType() != DHCP6.MsgType.REPLY.value()) {
642 // Update IP address only when we received REPLY message
643 return;
644 }
645 Optional<Dhcp6ClientIdOption> clientIdOption = dhcp6.getOptions()
646 .stream()
647 .filter(opt -> opt instanceof Dhcp6ClientIdOption)
648 .map(opt -> (Dhcp6ClientIdOption) opt)
649 .findFirst();
650
651 if (!clientIdOption.isPresent()) {
652 // invalid DHCPv6 option
653 log.warn("Can't find client ID from DHCPv6 {}", dhcp6);
654 return;
655 }
656
657 byte[] linkLayerAddr = clientIdOption.get().getDuid().getLinkLayerAddress();
658 if (linkLayerAddr == null || linkLayerAddr.length != 6) {
659 // No any mac address found
660 log.warn("Can't find client mac from option {}", clientIdOption);
661 return;
662 }
663 MacAddress clientMac = MacAddress.valueOf(linkLayerAddr);
664
665 // Extract IPv6 address from IA NA ot IA TA option
666 Optional<Dhcp6IaNaOption> iaNaOption = dhcp6.getOptions()
667 .stream()
668 .filter(opt -> opt instanceof Dhcp6IaNaOption)
669 .map(opt -> (Dhcp6IaNaOption) opt)
670 .findFirst();
671 Optional<Dhcp6IaTaOption> iaTaOption = dhcp6.getOptions()
672 .stream()
673 .filter(opt -> opt instanceof Dhcp6IaTaOption)
674 .map(opt -> (Dhcp6IaTaOption) opt)
675 .findFirst();
676 Optional<Dhcp6IaAddressOption> iaAddressOption;
677 if (iaNaOption.isPresent()) {
678 iaAddressOption = iaNaOption.get().getOptions().stream()
679 .filter(opt -> opt instanceof Dhcp6IaAddressOption)
680 .map(opt -> (Dhcp6IaAddressOption) opt)
681 .findFirst();
682 } else if (iaTaOption.isPresent()) {
683 iaAddressOption = iaTaOption.get().getOptions().stream()
684 .filter(opt -> opt instanceof Dhcp6IaAddressOption)
685 .map(opt -> (Dhcp6IaAddressOption) opt)
686 .findFirst();
687 } else {
688 iaAddressOption = Optional.empty();
689 }
690 if (iaAddressOption.isPresent()) {
691 Ip6Address ip = iaAddressOption.get().getIp6Address();
692 HostId hostId = HostId.hostId(clientMac, vlanId);
693 updateHostIp(hostId, ip);
694 } else {
695 log.warn("Can't find IPv6 address from DHCPv6 {}", dhcp6);
696 }
697 }
698
Yi Tseng7a38f9a2017-06-09 14:36:40 -0700699 private Optional<DHCP> findDhcp(Ethernet eth) {
700 IPacket pkt = eth.getPayload();
701 return Stream.of(pkt)
702 .filter(Objects::nonNull)
703 .filter(p -> p instanceof IPv4)
704 .map(IPacket::getPayload)
705 .filter(Objects::nonNull)
706 .filter(p -> p instanceof UDP)
707 .map(IPacket::getPayload)
708 .filter(Objects::nonNull)
709 .filter(p -> p instanceof DHCP)
710 .map(p -> (DHCP) p)
711 .findFirst();
712 }
Yi Tsengfcf5dce2017-07-26 14:30:41 -0700713
714 private Optional<DHCP6> findDhcp6(IPacket pkt) {
715 return Stream.of(pkt)
716 .filter(Objects::nonNull)
717 .filter(p -> p instanceof UDP)
718 .map(IPacket::getPayload)
719 .filter(Objects::nonNull)
720 .filter(p -> p instanceof DHCP6)
721 .map(p -> (DHCP6) p)
722 .findFirst();
723 }
alshabibe1cf87d2014-10-17 09:23:50 -0700724 }
Thomas Vachuska33601602014-11-19 03:32:15 -0800725
726 // Auxiliary listener to device events.
727 private class InternalDeviceListener implements DeviceListener {
728 @Override
729 public void event(DeviceEvent event) {
Madan Jampania3770c32015-12-11 12:07:41 -0800730 eventHandler.execute(() -> handleEvent(event));
731 }
732
733 private void handleEvent(DeviceEvent event) {
Pavlin Radoslavovd36a74b2015-01-09 11:59:07 -0800734 Device device = event.subject();
735 switch (event.type()) {
Thomas Vachuska27bee092015-06-23 19:03:10 -0700736 case DEVICE_ADDED:
737 break;
738 case DEVICE_AVAILABILITY_CHANGED:
Charles Chan888e20a2017-05-01 15:44:23 -0700739 if (hostRemovalEnabled && !deviceService.isAvailable(device.id())) {
740 processDeviceDown(device.id());
Thomas Vachuska27bee092015-06-23 19:03:10 -0700741 }
742 break;
743 case DEVICE_SUSPENDED:
744 case DEVICE_UPDATED:
745 // Nothing to do?
746 break;
747 case DEVICE_REMOVED:
748 if (hostRemovalEnabled) {
Charles Chan888e20a2017-05-01 15:44:23 -0700749 processDeviceDown(device.id());
Thomas Vachuska27bee092015-06-23 19:03:10 -0700750 }
751 break;
752 case PORT_ADDED:
753 break;
754 case PORT_UPDATED:
Charles Chan888e20a2017-05-01 15:44:23 -0700755 if (hostRemovalEnabled && !event.port().isEnabled()) {
756 processPortDown(new ConnectPoint(device.id(), event.port().number()));
Thomas Vachuska27bee092015-06-23 19:03:10 -0700757 }
758 break;
759 case PORT_REMOVED:
760 // Nothing to do?
761 break;
762 default:
763 break;
Thomas Vachuska33601602014-11-19 03:32:15 -0800764 }
765 }
766 }
767
Charles Chan888e20a2017-05-01 15:44:23 -0700768 /**
769 * When a device goes down, update the location of affected hosts.
770 *
771 * @param deviceId the device that goes down
772 */
773 private void processDeviceDown(DeviceId deviceId) {
774 hostService.getConnectedHosts(deviceId).forEach(affectedHost -> affectedHost.locations().stream()
775 .filter(hostLocation -> hostLocation.deviceId().equals(deviceId))
776 .forEach(affectedLocation ->
777 providerService.removeLocationFromHost(affectedHost.id(), affectedLocation))
778 );
779 }
780
781 /**
782 * When a port goes down, update the location of affected hosts.
783 *
784 * @param connectPoint the port that goes down
785 */
786 private void processPortDown(ConnectPoint connectPoint) {
787 hostService.getConnectedHosts(connectPoint).forEach(affectedHost ->
788 providerService.removeLocationFromHost(affectedHost.id(), new HostLocation(connectPoint, 0L))
789 );
Thomas Vachuska33601602014-11-19 03:32:15 -0800790 }
791
alshabibe1cf87d2014-10-17 09:23:50 -0700792}