Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 1 | /* |
Brian O'Connor | a09fe5b | 2017-08-03 21:12:30 -0700 | [diff] [blame] | 2 | * Copyright 2016-present Open Networking Foundation |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package org.onosproject.segmentrouting; |
| 18 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 19 | import com.google.common.cache.Cache; |
| 20 | import com.google.common.cache.CacheBuilder; |
| 21 | import com.google.common.cache.RemovalCause; |
| 22 | import com.google.common.cache.RemovalNotification; |
Saurav Das | 7bcbe70 | 2017-06-13 15:35:54 -0700 | [diff] [blame] | 23 | import com.google.common.collect.Sets; |
| 24 | |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 25 | import org.onlab.packet.IpPrefix; |
| 26 | import org.onlab.packet.MacAddress; |
Charles Chan | 7ffd81f | 2017-02-08 15:52:08 -0800 | [diff] [blame] | 27 | import org.onlab.packet.VlanId; |
Charles Chan | 9640c81 | 2017-08-23 13:55:39 -0700 | [diff] [blame] | 28 | import org.onosproject.net.ConnectPoint; |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 29 | import org.onosproject.net.HostLocation; |
| 30 | import org.onosproject.net.PortNumber; |
| 31 | import org.onosproject.net.host.HostEvent; |
Ray Milkey | 69ec871 | 2017-08-08 13:00:43 -0700 | [diff] [blame] | 32 | import org.onosproject.routeservice.ResolvedRoute; |
| 33 | import org.onosproject.routeservice.RouteEvent; |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 34 | import org.onosproject.net.DeviceId; |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 35 | import org.onosproject.routeservice.RouteInfo; |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 36 | import org.slf4j.Logger; |
| 37 | import org.slf4j.LoggerFactory; |
| 38 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 39 | import java.util.Collection; |
Charles Chan | b8664b8 | 2017-06-22 14:15:05 -0700 | [diff] [blame] | 40 | import java.util.Objects; |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 41 | import java.util.Optional; |
| 42 | import java.util.Set; |
| 43 | import java.util.concurrent.Executors; |
| 44 | import java.util.concurrent.TimeUnit; |
| 45 | import java.util.stream.Collectors; |
Charles Chan | b8664b8 | 2017-06-22 14:15:05 -0700 | [diff] [blame] | 46 | |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 47 | /** |
| 48 | * Handles RouteEvent and manages routing entries. |
| 49 | */ |
| 50 | public class RouteHandler { |
| 51 | private static final Logger log = LoggerFactory.getLogger(RouteHandler.class); |
| 52 | private final SegmentRoutingManager srManager; |
| 53 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 54 | private static final int WAIT_TIME_MS = 1000; |
| 55 | /** |
| 56 | * The routeEventCache is implemented to avoid race condition by giving more time to the |
| 57 | * underlying flow subsystem to process previous populateSubnet call. |
| 58 | */ |
| 59 | private Cache<IpPrefix, RouteEvent> routeEventCache = CacheBuilder.newBuilder() |
| 60 | .expireAfterWrite(WAIT_TIME_MS, TimeUnit.MILLISECONDS) |
| 61 | .removalListener((RemovalNotification<IpPrefix, RouteEvent> notification) -> { |
| 62 | IpPrefix prefix = notification.getKey(); |
| 63 | RouteEvent routeEvent = notification.getValue(); |
| 64 | RemovalCause cause = notification.getCause(); |
| 65 | log.debug("routeEventCache removal event. prefix={}, routeEvent={}, cause={}", |
| 66 | prefix, routeEvent, cause); |
| 67 | |
| 68 | switch (notification.getCause()) { |
| 69 | case REPLACED: |
| 70 | case EXPIRED: |
| 71 | dequeueRouteEvent(routeEvent); |
| 72 | break; |
| 73 | default: |
| 74 | break; |
| 75 | } |
| 76 | }).build(); |
| 77 | |
Charles Chan | 9640c81 | 2017-08-23 13:55:39 -0700 | [diff] [blame] | 78 | RouteHandler(SegmentRoutingManager srManager) { |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 79 | this.srManager = srManager; |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 80 | |
| 81 | Executors.newSingleThreadScheduledExecutor() |
| 82 | .scheduleAtFixedRate(routeEventCache::cleanUp, 0, WAIT_TIME_MS, TimeUnit.MILLISECONDS); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 83 | } |
| 84 | |
| 85 | protected void init(DeviceId deviceId) { |
Charles Chan | 9640c81 | 2017-08-23 13:55:39 -0700 | [diff] [blame] | 86 | srManager.routeService.getRouteTables().forEach(routeTableId -> |
| 87 | srManager.routeService.getRoutes(routeTableId).forEach(routeInfo -> |
| 88 | routeInfo.allRoutes().forEach(resolvedRoute -> |
| 89 | srManager.nextHopLocations(resolvedRoute).stream() |
| 90 | .filter(location -> deviceId.equals(location.deviceId())) |
| 91 | .forEach(location -> processRouteAddedInternal(resolvedRoute) |
| 92 | ) |
| 93 | ) |
| 94 | ) |
| 95 | ); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 96 | } |
| 97 | |
Charles Chan | 9640c81 | 2017-08-23 13:55:39 -0700 | [diff] [blame] | 98 | void processRouteAdded(RouteEvent event) { |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 99 | enqueueRouteEvent(event); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 100 | } |
| 101 | |
| 102 | private void processRouteAddedInternal(ResolvedRoute route) { |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 103 | processRouteAddedInternal(Sets.newHashSet(route)); |
| 104 | } |
| 105 | |
| 106 | private void processRouteAddedInternal(Collection<ResolvedRoute> routes) { |
Charles Chan | b8664b8 | 2017-06-22 14:15:05 -0700 | [diff] [blame] | 107 | if (!isReady()) { |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 108 | log.info("System is not ready. Skip adding route for {}", routes); |
Charles Chan | b8664b8 | 2017-06-22 14:15:05 -0700 | [diff] [blame] | 109 | return; |
| 110 | } |
| 111 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 112 | log.info("processRouteAddedInternal. routes={}", routes); |
Charles Chan | 9640c81 | 2017-08-23 13:55:39 -0700 | [diff] [blame] | 113 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 114 | Set<ConnectPoint> allLocations = Sets.newHashSet(); |
| 115 | Set<IpPrefix> allPrefixes = Sets.newHashSet(); |
| 116 | routes.forEach(route -> { |
| 117 | allLocations.addAll(srManager.nextHopLocations(route)); |
| 118 | allPrefixes.add(route.prefix()); |
| 119 | }); |
| 120 | log.debug("RouteAdded. populateSubnet {}, {}", allLocations, allPrefixes); |
| 121 | srManager.defaultRoutingHandler.populateSubnet(allLocations, allPrefixes); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 122 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 123 | routes.forEach(route -> { |
| 124 | IpPrefix prefix = route.prefix(); |
| 125 | MacAddress nextHopMac = route.nextHopMac(); |
| 126 | VlanId nextHopVlan = route.nextHopVlan(); |
| 127 | Set<ConnectPoint> locations = srManager.nextHopLocations(route); |
| 128 | |
| 129 | locations.forEach(location -> { |
| 130 | log.debug("RouteAdded. addSubnet {}, {}", location, prefix); |
| 131 | srManager.deviceConfiguration.addSubnet(location, prefix); |
| 132 | log.debug("RouteAdded populateRoute {}, {}, {}, {}", location, prefix, nextHopMac, nextHopVlan); |
| 133 | srManager.defaultRoutingHandler.populateRoute(location.deviceId(), prefix, |
| 134 | nextHopMac, nextHopVlan, location.port()); |
| 135 | }); |
| 136 | }); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 137 | } |
| 138 | |
Charles Chan | 9640c81 | 2017-08-23 13:55:39 -0700 | [diff] [blame] | 139 | void processRouteUpdated(RouteEvent event) { |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 140 | enqueueRouteEvent(event); |
| 141 | } |
| 142 | |
| 143 | void processAlternativeRoutesChanged(RouteEvent event) { |
| 144 | enqueueRouteEvent(event); |
| 145 | } |
| 146 | |
| 147 | private void processRouteUpdatedInternal(Set<ResolvedRoute> routes, Set<ResolvedRoute> oldRoutes) { |
| 148 | if (!isReady()) { |
| 149 | log.info("System is not ready. Skip updating route for {} -> {}", oldRoutes, routes); |
| 150 | return; |
| 151 | } |
| 152 | |
| 153 | log.info("processRouteUpdatedInternal. routes={}, oldRoutes={}", routes, oldRoutes); |
| 154 | |
| 155 | Set<ConnectPoint> allLocations = Sets.newHashSet(); |
| 156 | Set<IpPrefix> allPrefixes = Sets.newHashSet(); |
| 157 | routes.forEach(route -> { |
| 158 | allLocations.addAll(srManager.nextHopLocations(route)); |
| 159 | allPrefixes.add(route.prefix()); |
| 160 | }); |
| 161 | log.debug("RouteUpdated. populateSubnet {}, {}", allLocations, allPrefixes); |
| 162 | srManager.defaultRoutingHandler.populateSubnet(allLocations, allPrefixes); |
| 163 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 164 | Set<ResolvedRoute> toBeRemoved = Sets.difference(oldRoutes, routes).immutableCopy(); |
| 165 | Set<ResolvedRoute> toBeAdded = Sets.difference(routes, oldRoutes).immutableCopy(); |
| 166 | |
| 167 | toBeRemoved.forEach(route -> { |
| 168 | srManager.nextHopLocations(route).forEach(oldLocation -> { |
Charles Chan | 08d9132 | 2018-02-05 17:20:05 -0800 | [diff] [blame] | 169 | if (toBeAdded.stream().map(srManager::nextHopLocations) |
| 170 | .flatMap(Set::stream).map(ConnectPoint::deviceId) |
| 171 | .noneMatch(deviceId -> deviceId.equals(oldLocation.deviceId()))) { |
| 172 | IpPrefix prefix = route.prefix(); |
| 173 | log.debug("RouteUpdated. removeSubnet {}, {}", oldLocation, prefix); |
| 174 | srManager.deviceConfiguration.removeSubnet(oldLocation, prefix); |
| 175 | // We don't remove the flow on the old location in occasion of two next hops becoming one |
| 176 | // since the populateSubnet will point the old location to the new location via spine. |
| 177 | } |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 178 | }); |
| 179 | }); |
| 180 | |
| 181 | toBeAdded.forEach(route -> { |
| 182 | IpPrefix prefix = route.prefix(); |
| 183 | MacAddress nextHopMac = route.nextHopMac(); |
| 184 | VlanId nextHopVlan = route.nextHopVlan(); |
| 185 | Set<ConnectPoint> locations = srManager.nextHopLocations(route); |
| 186 | |
| 187 | locations.forEach(location -> { |
| 188 | log.debug("RouteUpdated. addSubnet {}, {}", location, prefix); |
| 189 | srManager.deviceConfiguration.addSubnet(location, prefix); |
| 190 | log.debug("RouteUpdated. populateRoute {}, {}, {}, {}", location, prefix, nextHopMac, nextHopVlan); |
| 191 | srManager.defaultRoutingHandler.populateRoute(location.deviceId(), prefix, |
| 192 | nextHopMac, nextHopVlan, location.port()); |
| 193 | }); |
| 194 | }); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 195 | } |
| 196 | |
Charles Chan | 9640c81 | 2017-08-23 13:55:39 -0700 | [diff] [blame] | 197 | void processRouteRemoved(RouteEvent event) { |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 198 | enqueueRouteEvent(event); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 199 | } |
| 200 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 201 | private void processRouteRemovedInternal(Collection<ResolvedRoute> routes) { |
Charles Chan | b8664b8 | 2017-06-22 14:15:05 -0700 | [diff] [blame] | 202 | if (!isReady()) { |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 203 | log.info("System is not ready. Skip removing route for {}", routes); |
Charles Chan | b8664b8 | 2017-06-22 14:15:05 -0700 | [diff] [blame] | 204 | return; |
| 205 | } |
| 206 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 207 | log.info("processRouteRemovedInternal. routes={}", routes); |
Charles Chan | 9640c81 | 2017-08-23 13:55:39 -0700 | [diff] [blame] | 208 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 209 | Set<IpPrefix> allPrefixes = Sets.newHashSet(); |
| 210 | routes.forEach(route -> { |
| 211 | allPrefixes.add(route.prefix()); |
| 212 | }); |
| 213 | log.debug("RouteRemoved. revokeSubnet {}", allPrefixes); |
| 214 | srManager.defaultRoutingHandler.revokeSubnet(allPrefixes); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 215 | |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 216 | routes.forEach(route -> { |
| 217 | IpPrefix prefix = route.prefix(); |
| 218 | MacAddress nextHopMac = route.nextHopMac(); |
| 219 | VlanId nextHopVlan = route.nextHopVlan(); |
| 220 | Set<ConnectPoint> locations = srManager.nextHopLocations(route); |
| 221 | |
| 222 | locations.forEach(location -> { |
| 223 | log.debug("RouteRemoved. removeSubnet {}, {}", location, prefix); |
| 224 | srManager.deviceConfiguration.removeSubnet(location, prefix); |
Charles Chan | f433f95 | 2018-03-15 16:41:10 -0700 | [diff] [blame] | 225 | // We don't need to call revokeRoute again since revokeSubnet will remove the prefix |
| 226 | // from all devices, including the ones that next hop attaches to. |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 227 | |
| 228 | // Also remove redirection flows on the pair device if exists. |
| 229 | Optional<DeviceId> pairDeviceId = srManager.getPairDeviceId(location.deviceId()); |
| 230 | Optional<PortNumber> pairLocalPort = srManager.getPairLocalPorts(location.deviceId()); |
| 231 | if (pairDeviceId.isPresent() && pairLocalPort.isPresent()) { |
| 232 | // NOTE: Since the pairLocalPort is trunk port, use assigned vlan of original port |
| 233 | // when the host is untagged |
| 234 | VlanId vlanId = Optional.ofNullable(srManager.getInternalVlanId(location)).orElse(nextHopVlan); |
| 235 | |
| 236 | log.debug("RouteRemoved. revokeRoute {}, {}, {}, {}", location, prefix, nextHopMac, nextHopVlan); |
| 237 | srManager.defaultRoutingHandler.revokeRoute(pairDeviceId.get(), prefix, |
| 238 | nextHopMac, vlanId, pairLocalPort.get()); |
| 239 | } |
| 240 | }); |
| 241 | }); |
| 242 | } |
| 243 | |
| 244 | void processHostMovedEvent(HostEvent event) { |
| 245 | log.info("processHostMovedEvent {}", event); |
| 246 | MacAddress hostMac = event.subject().mac(); |
| 247 | VlanId hostVlanId = event.subject().vlan(); |
| 248 | |
| 249 | affectedRoutes(hostMac, hostVlanId).forEach(affectedRoute -> { |
| 250 | IpPrefix prefix = affectedRoute.prefix(); |
| 251 | Set<HostLocation> prevLocations = event.prevSubject().locations(); |
| 252 | Set<HostLocation> newLocations = event.subject().locations(); |
| 253 | |
| 254 | // For each old location |
Charles Chan | 2ff1bac | 2018-03-29 16:03:41 -0700 | [diff] [blame] | 255 | Sets.difference(prevLocations, newLocations).forEach(prevLocation -> { |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 256 | // Redirect the flows to pair link if configured |
| 257 | // Note: Do not continue removing any rule |
| 258 | Optional<DeviceId> pairDeviceId = srManager.getPairDeviceId(prevLocation.deviceId()); |
| 259 | Optional<PortNumber> pairLocalPort = srManager.getPairLocalPorts(prevLocation.deviceId()); |
| 260 | if (pairDeviceId.isPresent() && pairLocalPort.isPresent() && newLocations.stream() |
| 261 | .anyMatch(location -> location.deviceId().equals(pairDeviceId.get()))) { |
| 262 | // NOTE: Since the pairLocalPort is trunk port, use assigned vlan of original port |
| 263 | // when the host is untagged |
| 264 | VlanId vlanId = Optional.ofNullable(srManager.getInternalVlanId(prevLocation)).orElse(hostVlanId); |
| 265 | log.debug("HostMoved. populateRoute {}, {}, {}, {}", prevLocation, prefix, hostMac, vlanId); |
| 266 | srManager.defaultRoutingHandler.populateRoute(prevLocation.deviceId(), prefix, |
| 267 | hostMac, vlanId, pairLocalPort.get()); |
| 268 | return; |
| 269 | } |
| 270 | |
| 271 | // No pair information supplied. Remove route |
| 272 | log.debug("HostMoved. revokeRoute {}, {}, {}, {}", prevLocation, prefix, hostMac, hostVlanId); |
| 273 | srManager.defaultRoutingHandler.revokeRoute(prevLocation.deviceId(), prefix, |
| 274 | hostMac, hostVlanId, prevLocation.port()); |
| 275 | }); |
| 276 | |
| 277 | // For each new location, add all new IPs. |
Charles Chan | 2ff1bac | 2018-03-29 16:03:41 -0700 | [diff] [blame] | 278 | Sets.difference(newLocations, prevLocations).forEach(newLocation -> { |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 279 | log.debug("HostMoved. populateRoute {}, {}, {}, {}", newLocation, prefix, hostMac, hostVlanId); |
| 280 | srManager.defaultRoutingHandler.populateRoute(newLocation.deviceId(), prefix, |
| 281 | hostMac, hostVlanId, newLocation.port()); |
| 282 | }); |
| 283 | |
| 284 | }); |
| 285 | } |
| 286 | |
| 287 | private Set<ResolvedRoute> affectedRoutes(MacAddress mac, VlanId vlanId) { |
| 288 | return srManager.routeService.getRouteTables().stream() |
| 289 | .map(routeTableId -> srManager.routeService.getRoutes(routeTableId)) |
| 290 | .flatMap(Collection::stream) |
| 291 | .map(RouteInfo::allRoutes) |
| 292 | .flatMap(Collection::stream) |
| 293 | .filter(resolvedRoute -> mac.equals(resolvedRoute.nextHopMac()) && |
| 294 | vlanId.equals(resolvedRoute.nextHopVlan())).collect(Collectors.toSet()); |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 295 | } |
Charles Chan | b8664b8 | 2017-06-22 14:15:05 -0700 | [diff] [blame] | 296 | |
| 297 | private boolean isReady() { |
| 298 | return Objects.nonNull(srManager.deviceConfiguration) && |
Charles Chan | 2fde6d4 | 2017-08-23 14:46:43 -0700 | [diff] [blame] | 299 | Objects.nonNull(srManager.defaultRoutingHandler); |
| 300 | } |
| 301 | |
| 302 | void enqueueRouteEvent(RouteEvent routeEvent) { |
| 303 | log.debug("Enqueue routeEvent {}", routeEvent); |
| 304 | routeEventCache.put(routeEvent.subject().prefix(), routeEvent); |
| 305 | } |
| 306 | |
| 307 | void dequeueRouteEvent(RouteEvent routeEvent) { |
| 308 | log.debug("Dequeue routeEvent {}", routeEvent); |
| 309 | switch (routeEvent.type()) { |
| 310 | case ROUTE_ADDED: |
| 311 | processRouteAddedInternal(routeEvent.alternatives()); |
| 312 | break; |
| 313 | case ROUTE_REMOVED: |
| 314 | processRouteRemovedInternal(routeEvent.alternatives()); |
| 315 | break; |
| 316 | case ROUTE_UPDATED: |
| 317 | case ALTERNATIVE_ROUTES_CHANGED: |
| 318 | processRouteUpdatedInternal(Sets.newHashSet(routeEvent.alternatives()), |
| 319 | Sets.newHashSet(routeEvent.prevAlternatives())); |
| 320 | break; |
| 321 | default: |
| 322 | break; |
| 323 | } |
Charles Chan | b8664b8 | 2017-06-22 14:15:05 -0700 | [diff] [blame] | 324 | } |
Charles Chan | 03a73e0 | 2016-10-24 14:52:01 -0700 | [diff] [blame] | 325 | } |