Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 1 | /* |
Brian O'Connor | a09fe5b | 2017-08-03 21:12:30 -0700 | [diff] [blame] | 2 | * Copyright 2016-present Open Networking Foundation |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package org.onosproject.dhcprelay; |
| 17 | |
| 18 | import java.nio.ByteBuffer; |
| 19 | import java.util.Collection; |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 20 | import java.util.Collections; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 21 | import java.util.Dictionary; |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 22 | import java.util.List; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 23 | import java.util.Objects; |
| 24 | import java.util.Optional; |
| 25 | import java.util.Set; |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 26 | import java.util.concurrent.atomic.AtomicInteger; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 27 | import java.util.stream.Stream; |
| 28 | |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 29 | import com.google.common.collect.HashMultimap; |
| 30 | import com.google.common.collect.ImmutableList; |
| 31 | import com.google.common.collect.Multimap; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 32 | import org.apache.felix.scr.annotations.Activate; |
| 33 | import org.apache.felix.scr.annotations.Component; |
| 34 | import org.apache.felix.scr.annotations.Deactivate; |
| 35 | import org.apache.felix.scr.annotations.Modified; |
| 36 | import org.apache.felix.scr.annotations.Property; |
| 37 | import org.apache.felix.scr.annotations.Reference; |
| 38 | import org.apache.felix.scr.annotations.ReferenceCardinality; |
| 39 | import org.apache.felix.scr.annotations.Service; |
| 40 | import org.onlab.packet.ARP; |
| 41 | import org.onlab.packet.DHCP; |
| 42 | import org.onlab.packet.DHCP6; |
| 43 | import org.onlab.packet.IPacket; |
| 44 | import org.onlab.packet.IPv6; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 45 | import org.onlab.packet.Ethernet; |
| 46 | import org.onlab.packet.IPv4; |
Yi Tseng | 4f2a046 | 2017-08-31 11:21:00 -0700 | [diff] [blame] | 47 | import org.onlab.packet.Ip4Address; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 48 | import org.onlab.packet.MacAddress; |
| 49 | import org.onlab.packet.TpPort; |
| 50 | import org.onlab.packet.UDP; |
| 51 | import org.onlab.packet.VlanId; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 52 | import org.onlab.util.Tools; |
| 53 | import org.onosproject.cfg.ComponentConfigService; |
| 54 | import org.onosproject.core.ApplicationId; |
| 55 | import org.onosproject.core.CoreService; |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 56 | import org.onosproject.dhcprelay.api.DhcpHandler; |
| 57 | import org.onosproject.dhcprelay.api.DhcpRelayService; |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 58 | import org.onosproject.dhcprelay.config.DefaultDhcpRelayConfig; |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 59 | import org.onosproject.dhcprelay.config.IgnoreDhcpConfig; |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 60 | import org.onosproject.dhcprelay.config.IndirectDhcpRelayConfig; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 61 | import org.onosproject.dhcprelay.store.DhcpRecord; |
| 62 | import org.onosproject.dhcprelay.store.DhcpRelayStore; |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 63 | import org.onosproject.net.Device; |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 64 | import org.onosproject.net.DeviceId; |
Yi Tseng | 4f2a046 | 2017-08-31 11:21:00 -0700 | [diff] [blame] | 65 | import org.onosproject.dhcprelay.config.DhcpServerConfig; |
| 66 | import org.onosproject.net.Host; |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 67 | import org.onosproject.net.behaviour.Pipeliner; |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 68 | import org.onosproject.net.config.Config; |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 69 | import org.onosproject.net.device.DeviceEvent; |
| 70 | import org.onosproject.net.device.DeviceListener; |
| 71 | import org.onosproject.net.device.DeviceService; |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 72 | import org.onosproject.net.flow.criteria.Criterion; |
| 73 | import org.onosproject.net.flow.criteria.UdpPortCriterion; |
| 74 | import org.onosproject.net.flowobjective.DefaultForwardingObjective; |
| 75 | import org.onosproject.net.flowobjective.FlowObjectiveService; |
| 76 | import org.onosproject.net.flowobjective.ForwardingObjective; |
| 77 | import org.onosproject.net.flowobjective.Objective; |
| 78 | import org.onosproject.net.flowobjective.ObjectiveContext; |
| 79 | import org.onosproject.net.flowobjective.ObjectiveError; |
Ray Milkey | facf286 | 2017-08-03 11:58:29 -0700 | [diff] [blame] | 80 | import org.onosproject.net.intf.Interface; |
| 81 | import org.onosproject.net.intf.InterfaceService; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 82 | import org.onosproject.net.ConnectPoint; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 83 | import org.onosproject.net.HostId; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 84 | import org.onosproject.net.config.ConfigFactory; |
| 85 | import org.onosproject.net.config.NetworkConfigEvent; |
| 86 | import org.onosproject.net.config.NetworkConfigListener; |
| 87 | import org.onosproject.net.config.NetworkConfigRegistry; |
| 88 | import org.onosproject.net.flow.DefaultTrafficSelector; |
| 89 | import org.onosproject.net.flow.DefaultTrafficTreatment; |
| 90 | import org.onosproject.net.flow.TrafficSelector; |
| 91 | import org.onosproject.net.flow.TrafficTreatment; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 92 | import org.onosproject.net.host.HostService; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 93 | import org.onosproject.net.packet.DefaultOutboundPacket; |
| 94 | import org.onosproject.net.packet.OutboundPacket; |
| 95 | import org.onosproject.net.packet.PacketContext; |
| 96 | import org.onosproject.net.packet.PacketPriority; |
| 97 | import org.onosproject.net.packet.PacketProcessor; |
| 98 | import org.onosproject.net.packet.PacketService; |
| 99 | import org.onosproject.net.provider.ProviderId; |
| 100 | import org.osgi.service.component.ComponentContext; |
| 101 | import org.slf4j.Logger; |
| 102 | import org.slf4j.LoggerFactory; |
| 103 | |
| 104 | import com.google.common.collect.ImmutableSet; |
| 105 | |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 106 | import static org.onosproject.net.config.basics.SubjectFactories.APP_SUBJECT_FACTORY; |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 107 | import static org.onosproject.net.flowobjective.Objective.Operation.ADD; |
| 108 | import static org.onosproject.net.flowobjective.Objective.Operation.REMOVE; |
| 109 | |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 110 | /** |
| 111 | * DHCP Relay Agent Application Component. |
| 112 | */ |
| 113 | @Component(immediate = true) |
| 114 | @Service |
| 115 | public class DhcpRelayManager implements DhcpRelayService { |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 116 | public static final String DHCP_RELAY_APP = "org.onosproject.dhcprelay"; |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 117 | public static final ProviderId PROVIDER_ID = new ProviderId("host", DHCP_RELAY_APP); |
Yi Tseng | 06799d6 | 2017-09-01 16:02:56 -0700 | [diff] [blame] | 118 | public static final String ROUTE_STORE_IMPL = |
| 119 | "org.onosproject.routeservice.store.RouteStoreImpl"; |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 120 | private static final TrafficSelector DHCP_SERVER_SELECTOR = DefaultTrafficSelector.builder() |
| 121 | .matchEthType(Ethernet.TYPE_IPV4) |
| 122 | .matchIPProtocol(IPv4.PROTOCOL_UDP) |
| 123 | .matchUdpDst(TpPort.tpPort(UDP.DHCP_SERVER_PORT)) |
| 124 | .build(); |
| 125 | private static final TrafficSelector DHCP_CLIENT_SELECTOR = DefaultTrafficSelector.builder() |
| 126 | .matchEthType(Ethernet.TYPE_IPV4) |
| 127 | .matchIPProtocol(IPv4.PROTOCOL_UDP) |
| 128 | .matchUdpDst(TpPort.tpPort(UDP.DHCP_CLIENT_PORT)) |
| 129 | .build(); |
| 130 | private static final TrafficSelector DHCP6_SERVER_SELECTOR = DefaultTrafficSelector.builder() |
| 131 | .matchEthType(Ethernet.TYPE_IPV6) |
| 132 | .matchIPProtocol(IPv4.PROTOCOL_UDP) |
| 133 | .matchUdpDst(TpPort.tpPort(UDP.DHCP_V6_SERVER_PORT)) |
| 134 | .build(); |
| 135 | private static final TrafficSelector DHCP6_CLIENT_SELECTOR = DefaultTrafficSelector.builder() |
| 136 | .matchEthType(Ethernet.TYPE_IPV6) |
| 137 | .matchIPProtocol(IPv4.PROTOCOL_UDP) |
| 138 | .matchUdpDst(TpPort.tpPort(UDP.DHCP_V6_CLIENT_PORT)) |
| 139 | .build(); |
| 140 | static final List<TrafficSelector> DHCP_SELECTORS = ImmutableList.of( |
| 141 | DHCP_SERVER_SELECTOR, |
| 142 | DHCP_CLIENT_SELECTOR, |
| 143 | DHCP6_SERVER_SELECTOR, |
| 144 | DHCP6_CLIENT_SELECTOR |
| 145 | ); |
| 146 | private static final TrafficSelector ARP_SELECTOR = DefaultTrafficSelector.builder() |
| 147 | .matchEthType(Ethernet.TYPE_ARP) |
| 148 | .build(); |
| 149 | private static final int IGNORE_CONTROL_PRIORITY = PacketPriority.CONTROL.priorityValue() + 1000; |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 150 | private final Logger log = LoggerFactory.getLogger(getClass()); |
| 151 | private final InternalConfigListener cfgListener = new InternalConfigListener(); |
| 152 | |
| 153 | private final Set<ConfigFactory> factories = ImmutableSet.of( |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 154 | new ConfigFactory<ApplicationId, DefaultDhcpRelayConfig>(APP_SUBJECT_FACTORY, |
| 155 | DefaultDhcpRelayConfig.class, |
| 156 | DefaultDhcpRelayConfig.KEY, |
| 157 | true) { |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 158 | @Override |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 159 | public DefaultDhcpRelayConfig createConfig() { |
| 160 | return new DefaultDhcpRelayConfig(); |
| 161 | } |
| 162 | }, |
| 163 | new ConfigFactory<ApplicationId, IndirectDhcpRelayConfig>(APP_SUBJECT_FACTORY, |
| 164 | IndirectDhcpRelayConfig.class, |
| 165 | IndirectDhcpRelayConfig.KEY, |
| 166 | true) { |
| 167 | @Override |
| 168 | public IndirectDhcpRelayConfig createConfig() { |
| 169 | return new IndirectDhcpRelayConfig(); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 170 | } |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 171 | }, |
| 172 | new ConfigFactory<ApplicationId, IgnoreDhcpConfig>(APP_SUBJECT_FACTORY, |
| 173 | IgnoreDhcpConfig.class, |
| 174 | IgnoreDhcpConfig.KEY, |
| 175 | true) { |
| 176 | @Override |
| 177 | public IgnoreDhcpConfig createConfig() { |
| 178 | return new IgnoreDhcpConfig(); |
| 179 | } |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 180 | } |
| 181 | ); |
| 182 | |
| 183 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 184 | protected NetworkConfigRegistry cfgService; |
| 185 | |
| 186 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 187 | protected CoreService coreService; |
| 188 | |
| 189 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 190 | protected PacketService packetService; |
| 191 | |
| 192 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 193 | protected HostService hostService; |
| 194 | |
| 195 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 196 | protected InterfaceService interfaceService; |
| 197 | |
| 198 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 199 | protected DhcpRelayStore dhcpRelayStore; |
| 200 | |
| 201 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 202 | protected ComponentConfigService compCfgService; |
| 203 | |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 204 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 205 | protected FlowObjectiveService flowObjectiveService; |
| 206 | |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 207 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 208 | protected DeviceService deviceService; |
| 209 | |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 210 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY, |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 211 | target = "(version=4)") |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 212 | protected DhcpHandler v4Handler; |
| 213 | |
| 214 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY, |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 215 | target = "(version=6)") |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 216 | protected DhcpHandler v6Handler; |
| 217 | |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 218 | @Property(name = "arpEnabled", boolValue = true, |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 219 | label = "Enable Address resolution protocol") |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 220 | protected boolean arpEnabled = true; |
| 221 | |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 222 | protected Multimap<DeviceId, VlanId> ignoredVlans = HashMultimap.create(); |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 223 | protected DeviceListener deviceListener = new InternalDeviceListener(); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 224 | private DhcpRelayPacketProcessor dhcpRelayPacketProcessor = new DhcpRelayPacketProcessor(); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 225 | private ApplicationId appId; |
| 226 | |
| 227 | @Activate |
| 228 | protected void activate(ComponentContext context) { |
| 229 | //start the dhcp relay agent |
| 230 | appId = coreService.registerApplication(DHCP_RELAY_APP); |
| 231 | |
| 232 | cfgService.addListener(cfgListener); |
| 233 | factories.forEach(cfgService::registerConfigFactory); |
| 234 | //update the dhcp server configuration. |
| 235 | updateConfig(); |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 236 | |
| 237 | //add the packet processor |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 238 | packetService.addProcessor(dhcpRelayPacketProcessor, PacketProcessor.director(0)); |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 239 | |
| 240 | // listen host event for dhcp server or the gateway |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 241 | requestDhcpPackets(); |
| 242 | modified(context); |
| 243 | |
Yi Tseng | 06799d6 | 2017-09-01 16:02:56 -0700 | [diff] [blame] | 244 | // Enable distribute route store |
| 245 | compCfgService.preSetProperty(ROUTE_STORE_IMPL, |
| 246 | "distributed", Boolean.TRUE.toString()); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 247 | compCfgService.registerProperties(getClass()); |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 248 | |
| 249 | deviceService.addListener(deviceListener); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 250 | log.info("DHCP-RELAY Started"); |
| 251 | } |
| 252 | |
| 253 | @Deactivate |
| 254 | protected void deactivate() { |
| 255 | cfgService.removeListener(cfgListener); |
| 256 | factories.forEach(cfgService::unregisterConfigFactory); |
| 257 | packetService.removeProcessor(dhcpRelayPacketProcessor); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 258 | cancelDhcpPackets(); |
| 259 | cancelArpPackets(); |
Yi Tseng | dfa8ee2 | 2017-08-07 12:45:01 -0700 | [diff] [blame] | 260 | compCfgService.unregisterProperties(getClass(), false); |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 261 | deviceService.removeListener(deviceListener); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 262 | log.info("DHCP-RELAY Stopped"); |
| 263 | } |
| 264 | |
| 265 | @Modified |
| 266 | protected void modified(ComponentContext context) { |
| 267 | Dictionary<?, ?> properties = context.getProperties(); |
| 268 | Boolean flag; |
| 269 | |
| 270 | flag = Tools.isPropertyEnabled(properties, "arpEnabled"); |
| 271 | if (flag != null) { |
| 272 | arpEnabled = flag; |
| 273 | log.info("Address resolution protocol is {}", |
| 274 | arpEnabled ? "enabled" : "disabled"); |
| 275 | } |
| 276 | |
| 277 | if (arpEnabled) { |
| 278 | requestArpPackets(); |
| 279 | } else { |
| 280 | cancelArpPackets(); |
| 281 | } |
| 282 | } |
| 283 | |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 284 | /** |
| 285 | * Updates DHCP relay app configuration. |
| 286 | */ |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 287 | private void updateConfig() { |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 288 | DefaultDhcpRelayConfig defaultConfig = |
| 289 | cfgService.getConfig(appId, DefaultDhcpRelayConfig.class); |
| 290 | IndirectDhcpRelayConfig indirectConfig = |
| 291 | cfgService.getConfig(appId, IndirectDhcpRelayConfig.class); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 292 | IgnoreDhcpConfig ignoreDhcpConfig = |
| 293 | cfgService.getConfig(appId, IgnoreDhcpConfig.class); |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 294 | |
| 295 | if (defaultConfig != null) { |
| 296 | updateConfig(defaultConfig); |
| 297 | } |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 298 | if (indirectConfig != null) { |
| 299 | updateConfig(indirectConfig); |
| 300 | } |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 301 | if (ignoreDhcpConfig != null) { |
| 302 | updateConfig(ignoreDhcpConfig); |
| 303 | } |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 304 | } |
| 305 | |
| 306 | /** |
| 307 | * Updates DHCP relay app configuration with given configuration. |
| 308 | * |
| 309 | * @param config the configuration ot update |
| 310 | */ |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 311 | protected void updateConfig(Config config) { |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 312 | if (config instanceof IndirectDhcpRelayConfig) { |
| 313 | IndirectDhcpRelayConfig indirectConfig = (IndirectDhcpRelayConfig) config; |
| 314 | v4Handler.setIndirectDhcpServerConfigs(indirectConfig.dhcpServerConfigs()); |
| 315 | v6Handler.setIndirectDhcpServerConfigs(indirectConfig.dhcpServerConfigs()); |
Yi Tseng | 3df7f9d | 2017-08-17 13:08:31 -0700 | [diff] [blame] | 316 | } else if (config instanceof DefaultDhcpRelayConfig) { |
| 317 | DefaultDhcpRelayConfig defaultConfig = (DefaultDhcpRelayConfig) config; |
| 318 | v4Handler.setDefaultDhcpServerConfigs(defaultConfig.dhcpServerConfigs()); |
| 319 | v6Handler.setDefaultDhcpServerConfigs(defaultConfig.dhcpServerConfigs()); |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 320 | } |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 321 | if (config instanceof IgnoreDhcpConfig) { |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 322 | updateIgnoreVlanRules((IgnoreDhcpConfig) config); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 323 | } |
| 324 | } |
| 325 | |
| 326 | protected void removeConfig(Config config) { |
| 327 | if (config instanceof IndirectDhcpRelayConfig) { |
| 328 | v4Handler.setIndirectDhcpServerConfigs(Collections.emptyList()); |
| 329 | v6Handler.setIndirectDhcpServerConfigs(Collections.emptyList()); |
| 330 | } else if (config instanceof DefaultDhcpRelayConfig) { |
| 331 | v4Handler.setDefaultDhcpServerConfigs(Collections.emptyList()); |
| 332 | v6Handler.setDefaultDhcpServerConfigs(Collections.emptyList()); |
| 333 | } |
| 334 | if (config instanceof IgnoreDhcpConfig) { |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 335 | ignoredVlans.forEach(((deviceId, vlanId) -> { |
| 336 | processIgnoreVlanRule(deviceId, vlanId, REMOVE); |
| 337 | })); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 338 | } |
| 339 | } |
| 340 | |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 341 | private void updateIgnoreVlanRules(IgnoreDhcpConfig config) { |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 342 | config.ignoredVlans().forEach((deviceId, vlanId) -> { |
| 343 | if (ignoredVlans.get(deviceId).contains(vlanId)) { |
| 344 | // don't need to process if it already ignored |
| 345 | return; |
| 346 | } |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 347 | processIgnoreVlanRule(deviceId, vlanId, ADD); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 348 | }); |
| 349 | |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 350 | ignoredVlans.forEach((deviceId, vlanId) -> { |
| 351 | if (!config.ignoredVlans().get(deviceId).contains(vlanId)) { |
| 352 | // not contains in new config, remove it |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 353 | processIgnoreVlanRule(deviceId, vlanId, REMOVE); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 354 | } |
| 355 | }); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 356 | } |
| 357 | |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 358 | /** |
| 359 | * Process the ignore rules. |
| 360 | * |
| 361 | * @param deviceId the device id |
| 362 | * @param vlanId the vlan to be ignored |
| 363 | * @param op the operation, ADD to install; REMOVE to uninstall rules |
| 364 | */ |
| 365 | private void processIgnoreVlanRule(DeviceId deviceId, VlanId vlanId, Objective.Operation op) { |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 366 | TrafficTreatment dropTreatment = DefaultTrafficTreatment.emptyTreatment(); |
| 367 | dropTreatment.clearedDeferred(); |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 368 | AtomicInteger installedCount = new AtomicInteger(DHCP_SELECTORS.size()); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 369 | DHCP_SELECTORS.forEach(trafficSelector -> { |
| 370 | UdpPortCriterion udpDst = (UdpPortCriterion) trafficSelector.getCriterion(Criterion.Type.UDP_DST); |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 371 | int udpDstPort = udpDst.udpPort().toInt(); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 372 | TrafficSelector selector = DefaultTrafficSelector.builder(trafficSelector) |
| 373 | .matchVlanId(vlanId) |
| 374 | .build(); |
| 375 | |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 376 | ForwardingObjective.Builder builder = DefaultForwardingObjective.builder() |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 377 | .withFlag(ForwardingObjective.Flag.VERSATILE) |
| 378 | .withSelector(selector) |
| 379 | .withPriority(IGNORE_CONTROL_PRIORITY) |
| 380 | .withTreatment(dropTreatment) |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 381 | .fromApp(appId); |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 382 | |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 383 | |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 384 | ObjectiveContext objectiveContext = new ObjectiveContext() { |
| 385 | @Override |
| 386 | public void onSuccess(Objective objective) { |
| 387 | log.info("Ignore rule {} (Vlan id {}, device {}, UDP dst {})", |
| 388 | op, vlanId, deviceId, udpDstPort); |
| 389 | int countDown = installedCount.decrementAndGet(); |
| 390 | if (countDown != 0) { |
| 391 | return; |
| 392 | } |
| 393 | switch (op) { |
| 394 | case ADD: |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 395 | |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 396 | ignoredVlans.put(deviceId, vlanId); |
| 397 | break; |
| 398 | case REMOVE: |
| 399 | ignoredVlans.remove(deviceId, vlanId); |
| 400 | break; |
| 401 | default: |
| 402 | log.warn("Unsupported objective operation {}", op); |
| 403 | break; |
| 404 | } |
| 405 | } |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 406 | |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 407 | @Override |
| 408 | public void onError(Objective objective, ObjectiveError error) { |
| 409 | log.warn("Can't {} ignore rule (vlan id {}, udp dst {}, device {}) due to {}", |
| 410 | op, vlanId, udpDstPort, deviceId, error); |
| 411 | } |
| 412 | }; |
| 413 | |
| 414 | ForwardingObjective fwd; |
| 415 | switch (op) { |
| 416 | case ADD: |
| 417 | fwd = builder.add(objectiveContext); |
| 418 | break; |
| 419 | case REMOVE: |
| 420 | fwd = builder.remove(objectiveContext); |
| 421 | break; |
| 422 | default: |
| 423 | log.warn("Unsupported objective operation {}", op); |
| 424 | return; |
| 425 | } |
| 426 | |
| 427 | Device device = deviceService.getDevice(deviceId); |
| 428 | if (device == null || !device.is(Pipeliner.class)) { |
| 429 | log.warn("Device {} is not available now, wait until device is available", deviceId); |
| 430 | return; |
| 431 | } |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 432 | flowObjectiveService.apply(deviceId, fwd); |
| 433 | }); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 434 | } |
| 435 | |
| 436 | /** |
| 437 | * Request DHCP packet in via PacketService. |
| 438 | */ |
| 439 | private void requestDhcpPackets() { |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 440 | DHCP_SELECTORS.forEach(trafficSelector -> { |
| 441 | packetService.requestPackets(trafficSelector, PacketPriority.CONTROL, appId); |
| 442 | }); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 443 | } |
| 444 | |
| 445 | /** |
| 446 | * Cancel requested DHCP packets in via packet service. |
| 447 | */ |
| 448 | private void cancelDhcpPackets() { |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 449 | DHCP_SELECTORS.forEach(trafficSelector -> { |
| 450 | packetService.cancelPackets(trafficSelector, PacketPriority.CONTROL, appId); |
| 451 | }); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 452 | } |
| 453 | |
| 454 | /** |
| 455 | * Request ARP packet in via PacketService. |
| 456 | */ |
| 457 | private void requestArpPackets() { |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 458 | packetService.requestPackets(ARP_SELECTOR, PacketPriority.CONTROL, appId); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 459 | } |
| 460 | |
| 461 | /** |
| 462 | * Cancel requested ARP packets in via packet service. |
| 463 | */ |
| 464 | private void cancelArpPackets() { |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 465 | packetService.cancelPackets(ARP_SELECTOR, PacketPriority.CONTROL, appId); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 466 | } |
| 467 | |
| 468 | @Override |
| 469 | public Optional<DhcpRecord> getDhcpRecord(HostId hostId) { |
| 470 | return dhcpRelayStore.getDhcpRecord(hostId); |
| 471 | } |
| 472 | |
| 473 | @Override |
| 474 | public Collection<DhcpRecord> getDhcpRecords() { |
| 475 | return dhcpRelayStore.getDhcpRecords(); |
| 476 | } |
| 477 | |
Yi Tseng | 13a41a1 | 2017-07-26 13:45:01 -0700 | [diff] [blame] | 478 | @Override |
| 479 | public Optional<MacAddress> getDhcpServerMacAddress() { |
Yi Tseng | 4f2a046 | 2017-08-31 11:21:00 -0700 | [diff] [blame] | 480 | // TODO: depreated it |
| 481 | DefaultDhcpRelayConfig config = cfgService.getConfig(appId, DefaultDhcpRelayConfig.class); |
| 482 | DhcpServerConfig serverConfig = config.dhcpServerConfigs().get(0); |
| 483 | Ip4Address serverip = serverConfig.getDhcpServerIp4().get(); |
| 484 | return hostService.getHostsByIp(serverip) |
| 485 | .stream() |
| 486 | .map(Host::mac) |
| 487 | .findFirst(); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 488 | } |
| 489 | |
| 490 | /** |
| 491 | * Gets DHCP data from a packet. |
| 492 | * |
| 493 | * @param packet the packet |
| 494 | * @return the DHCP data; empty if it is not a DHCP packet |
| 495 | */ |
| 496 | private Optional<DHCP> findDhcp(Ethernet packet) { |
| 497 | return Stream.of(packet) |
| 498 | .filter(Objects::nonNull) |
| 499 | .map(Ethernet::getPayload) |
| 500 | .filter(p -> p instanceof IPv4) |
| 501 | .map(IPacket::getPayload) |
| 502 | .filter(Objects::nonNull) |
| 503 | .filter(p -> p instanceof UDP) |
| 504 | .map(IPacket::getPayload) |
| 505 | .filter(Objects::nonNull) |
| 506 | .filter(p -> p instanceof DHCP) |
| 507 | .map(p -> (DHCP) p) |
| 508 | .findFirst(); |
| 509 | } |
| 510 | |
| 511 | /** |
| 512 | * Gets DHCPv6 data from a packet. |
| 513 | * |
| 514 | * @param packet the packet |
| 515 | * @return the DHCPv6 data; empty if it is not a DHCPv6 packet |
| 516 | */ |
| 517 | private Optional<DHCP6> findDhcp6(Ethernet packet) { |
| 518 | return Stream.of(packet) |
| 519 | .filter(Objects::nonNull) |
| 520 | .map(Ethernet::getPayload) |
| 521 | .filter(p -> p instanceof IPv6) |
| 522 | .map(IPacket::getPayload) |
| 523 | .filter(Objects::nonNull) |
| 524 | .filter(p -> p instanceof UDP) |
| 525 | .map(IPacket::getPayload) |
| 526 | .filter(Objects::nonNull) |
| 527 | .filter(p -> p instanceof DHCP6) |
| 528 | .map(p -> (DHCP6) p) |
| 529 | .findFirst(); |
| 530 | } |
| 531 | |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 532 | |
| 533 | private class DhcpRelayPacketProcessor implements PacketProcessor { |
| 534 | |
| 535 | @Override |
| 536 | public void process(PacketContext context) { |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 537 | // process the packet and get the payload |
| 538 | Ethernet packet = context.inPacket().parsed(); |
| 539 | if (packet == null) { |
| 540 | return; |
| 541 | } |
| 542 | |
| 543 | findDhcp(packet).ifPresent(dhcpPayload -> { |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 544 | v4Handler.processDhcpPacket(context, dhcpPayload); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 545 | }); |
| 546 | |
| 547 | findDhcp6(packet).ifPresent(dhcp6Payload -> { |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 548 | v6Handler.processDhcpPacket(context, dhcp6Payload); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 549 | }); |
| 550 | |
| 551 | if (packet.getEtherType() == Ethernet.TYPE_ARP && arpEnabled) { |
| 552 | ARP arpPacket = (ARP) packet.getPayload(); |
| 553 | VlanId vlanId = VlanId.vlanId(packet.getVlanID()); |
| 554 | Set<Interface> interfaces = interfaceService. |
| 555 | getInterfacesByPort(context.inPacket().receivedFrom()); |
| 556 | //ignore the packets if dhcp server interface is not configured on onos. |
| 557 | if (interfaces.isEmpty()) { |
| 558 | log.warn("server virtual interface not configured"); |
| 559 | return; |
| 560 | } |
| 561 | if ((arpPacket.getOpCode() != ARP.OP_REQUEST)) { |
| 562 | // handle request only |
| 563 | return; |
| 564 | } |
| 565 | MacAddress interfaceMac = interfaces.stream() |
| 566 | .filter(iface -> iface.vlan().equals(vlanId)) |
| 567 | .map(Interface::mac) |
| 568 | .filter(mac -> !mac.equals(MacAddress.NONE)) |
| 569 | .findFirst() |
Yi Tseng | 5130129 | 2017-07-28 13:02:59 -0700 | [diff] [blame] | 570 | .orElse(MacAddress.ONOS); |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 571 | if (interfaceMac == null) { |
| 572 | // can't find interface mac address |
| 573 | return; |
| 574 | } |
| 575 | processArpPacket(context, packet, interfaceMac); |
| 576 | } |
| 577 | } |
| 578 | |
| 579 | /** |
| 580 | * Processes the ARP Payload and initiates a reply to the client. |
| 581 | * |
| 582 | * @param context the packet context |
| 583 | * @param packet the ethernet payload |
| 584 | * @param replyMac mac address to be replied |
| 585 | */ |
| 586 | private void processArpPacket(PacketContext context, Ethernet packet, MacAddress replyMac) { |
| 587 | ARP arpPacket = (ARP) packet.getPayload(); |
| 588 | ARP arpReply = (ARP) arpPacket.clone(); |
| 589 | arpReply.setOpCode(ARP.OP_REPLY); |
| 590 | |
| 591 | arpReply.setTargetProtocolAddress(arpPacket.getSenderProtocolAddress()); |
| 592 | arpReply.setTargetHardwareAddress(arpPacket.getSenderHardwareAddress()); |
| 593 | arpReply.setSenderProtocolAddress(arpPacket.getTargetProtocolAddress()); |
| 594 | arpReply.setSenderHardwareAddress(replyMac.toBytes()); |
| 595 | |
| 596 | // Ethernet Frame. |
| 597 | Ethernet ethReply = new Ethernet(); |
| 598 | ethReply.setSourceMACAddress(replyMac.toBytes()); |
| 599 | ethReply.setDestinationMACAddress(packet.getSourceMAC()); |
| 600 | ethReply.setEtherType(Ethernet.TYPE_ARP); |
| 601 | ethReply.setVlanID(packet.getVlanID()); |
| 602 | ethReply.setPayload(arpReply); |
| 603 | |
| 604 | ConnectPoint targetPort = context.inPacket().receivedFrom(); |
| 605 | TrafficTreatment t = DefaultTrafficTreatment.builder() |
| 606 | .setOutput(targetPort.port()).build(); |
| 607 | OutboundPacket o = new DefaultOutboundPacket( |
| 608 | targetPort.deviceId(), t, ByteBuffer.wrap(ethReply.serialize())); |
| 609 | if (log.isTraceEnabled()) { |
| 610 | log.trace("Relaying ARP packet {} to {}", packet, targetPort); |
| 611 | } |
| 612 | packetService.emit(o); |
| 613 | } |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 614 | } |
| 615 | |
| 616 | /** |
| 617 | * Listener for network config events. |
| 618 | */ |
| 619 | private class InternalConfigListener implements NetworkConfigListener { |
| 620 | @Override |
| 621 | public void event(NetworkConfigEvent event) { |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 622 | switch (event.type()) { |
| 623 | case CONFIG_UPDATED: |
| 624 | case CONFIG_ADDED: |
| 625 | event.config().ifPresent(config -> { |
| 626 | updateConfig(config); |
| 627 | log.info("{} updated", config.getClass().getSimpleName()); |
| 628 | }); |
| 629 | break; |
| 630 | case CONFIG_REMOVED: |
| 631 | event.prevConfig().ifPresent(config -> { |
| 632 | removeConfig(config); |
| 633 | log.info("{} removed", config.getClass().getSimpleName()); |
| 634 | }); |
| 635 | break; |
| 636 | default: |
| 637 | log.warn("Unsupported event type {}", event.type()); |
| 638 | break; |
Yi Tseng | 483ac6f | 2017-08-02 15:03:31 -0700 | [diff] [blame] | 639 | } |
Yi Tseng | 51f1be9 | 2017-09-01 17:24:57 -0700 | [diff] [blame] | 640 | |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 641 | } |
| 642 | } |
Yi Tseng | 127ffe5 | 2017-09-12 15:55:17 -0700 | [diff] [blame] | 643 | |
| 644 | private class InternalDeviceListener implements DeviceListener { |
| 645 | |
| 646 | @Override |
| 647 | public void event(DeviceEvent event) { |
| 648 | Device device = event.subject(); |
| 649 | switch (event.type()) { |
| 650 | case DEVICE_ADDED: |
| 651 | deviceAdd(device.id()); |
| 652 | break; |
| 653 | case DEVICE_REMOVED: |
| 654 | ignoredVlans.removeAll(device.id()); |
| 655 | break; |
| 656 | case DEVICE_AVAILABILITY_CHANGED: |
| 657 | deviceAvailabilityChanged(device); |
| 658 | |
| 659 | default: |
| 660 | break; |
| 661 | } |
| 662 | } |
| 663 | |
| 664 | private void deviceAvailabilityChanged(Device device) { |
| 665 | if (deviceService.isAvailable(device.id())) { |
| 666 | deviceAdd(device.id()); |
| 667 | } else { |
| 668 | ignoredVlans.removeAll(device.id()); |
| 669 | } |
| 670 | } |
| 671 | |
| 672 | private void deviceAdd(DeviceId deviceId) { |
| 673 | IgnoreDhcpConfig config = cfgService.getConfig(appId, IgnoreDhcpConfig.class); |
| 674 | Collection<VlanId> vlanIds = config.ignoredVlans().get(deviceId); |
| 675 | vlanIds.forEach(vlanId -> { |
| 676 | processIgnoreVlanRule(deviceId, vlanId, ADD); |
| 677 | }); |
| 678 | } |
| 679 | } |
Yi Tseng | 7a38f9a | 2017-06-09 14:36:40 -0700 | [diff] [blame] | 680 | } |