blob: 808f7c6ffee140e47f67f98a6b32e0489ad32f3a [file] [log] [blame]
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001/*
2 * Copyright 2015 Open Networking Laboratory
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package org.onosproject.cordvtn;
17
Hyunsun Moonc71231d2015-12-16 20:53:23 -080018import com.google.common.collect.Lists;
19import com.google.common.collect.Maps;
20import org.onlab.packet.Ethernet;
Hyunsun Moon4a915152016-01-14 16:56:26 -080021import org.onlab.packet.IPv4;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080022import org.onlab.packet.Ip4Address;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080023import org.onlab.packet.Ip4Prefix;
24import org.onlab.packet.IpAddress;
25import org.onlab.packet.IpPrefix;
26import org.onlab.packet.MacAddress;
Hyunsun Moon4a915152016-01-14 16:56:26 -080027import org.onlab.packet.TpPort;
Hyunsun Moon6d247342016-02-12 12:48:47 -080028import org.onlab.packet.VlanId;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080029import org.onlab.util.ItemNotFoundException;
30import org.onosproject.core.ApplicationId;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080031import org.onosproject.core.DefaultGroupId;
32import org.onosproject.core.GroupId;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080033import org.onosproject.net.Device;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080034import org.onosproject.net.DeviceId;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080035import org.onosproject.net.Host;
Hyunsun Moon5a4346f2016-01-15 11:41:14 -080036import org.onosproject.net.Port;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080037import org.onosproject.net.PortNumber;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080038import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080039import org.onosproject.net.device.DeviceService;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080040import org.onosproject.net.driver.DefaultDriverData;
41import org.onosproject.net.driver.DefaultDriverHandler;
42import org.onosproject.net.driver.Driver;
43import org.onosproject.net.driver.DriverHandler;
44import org.onosproject.net.driver.DriverService;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080045import org.onosproject.net.flow.DefaultFlowRule;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080046import org.onosproject.net.flow.DefaultTrafficSelector;
47import org.onosproject.net.flow.DefaultTrafficTreatment;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080048import org.onosproject.net.flow.FlowRule;
49import org.onosproject.net.flow.FlowRuleOperations;
50import org.onosproject.net.flow.FlowRuleOperationsContext;
51import org.onosproject.net.flow.FlowRuleService;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080052import org.onosproject.net.flow.TrafficSelector;
53import org.onosproject.net.flow.TrafficTreatment;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080054import org.onosproject.net.flow.criteria.Criterion;
55import org.onosproject.net.flow.criteria.EthCriterion;
56import org.onosproject.net.flow.criteria.IPCriterion;
57import org.onosproject.net.flow.criteria.PortCriterion;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080058import org.onosproject.net.flow.instructions.ExtensionPropertyException;
59import org.onosproject.net.flow.instructions.ExtensionTreatment;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080060import org.onosproject.net.flow.instructions.Instruction;
61import org.onosproject.net.flow.instructions.Instructions;
Hyunsun Moon6d247342016-02-12 12:48:47 -080062import org.onosproject.net.flow.instructions.L2ModificationInstruction;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080063import org.onosproject.net.flow.instructions.L2ModificationInstruction.ModEtherInstruction;
64import org.onosproject.net.group.DefaultGroupBucket;
65import org.onosproject.net.group.DefaultGroupDescription;
66import org.onosproject.net.group.DefaultGroupKey;
67import org.onosproject.net.group.Group;
68import org.onosproject.net.group.GroupBucket;
69import org.onosproject.net.group.GroupBuckets;
70import org.onosproject.net.group.GroupDescription;
71import org.onosproject.net.group.GroupKey;
72import org.onosproject.net.group.GroupService;
sangho93447f12016-02-24 00:33:22 +090073import org.onosproject.openstackinterface.OpenstackNetwork;
74import org.onosproject.openstackinterface.OpenstackSubnet;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080075import org.slf4j.Logger;
76
Hyunsun Moon4161e6f2016-01-07 01:32:31 -080077import java.util.ArrayList;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080078import java.util.List;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080079import java.util.Map;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080080import java.util.Objects;
81import java.util.Set;
82import java.util.stream.Collectors;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080083
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080084import static com.google.common.base.Preconditions.checkNotNull;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080085import static org.onosproject.net.Device.Type.SWITCH;
86import static org.onosproject.net.flow.criteria.Criterion.Type.IN_PORT;
87import static org.onosproject.net.flow.criteria.Criterion.Type.IPV4_DST;
88import static org.onosproject.net.flow.criteria.Criterion.Type.IPV4_SRC;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080089import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST;
Hyunsun Moonc71231d2015-12-16 20:53:23 -080090import static org.onosproject.net.flow.instructions.L2ModificationInstruction.L2SubType.ETH_DST;
Hyunsun Moon6d247342016-02-12 12:48:47 -080091import static org.onosproject.net.flow.instructions.L2ModificationInstruction.L2SubType.VLAN_PUSH;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080092import static org.slf4j.LoggerFactory.getLogger;
93
94/**
Hyunsun Moonc71231d2015-12-16 20:53:23 -080095 * Populates rules for CORD VTN service.
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080096 */
Hyunsun Moonc71231d2015-12-16 20:53:23 -080097public class CordVtnRuleInstaller {
98
Hyunsun Moon9f0814b2015-11-04 17:34:35 -080099 protected final Logger log = getLogger(getClass());
100
Hyunsun Moon4a915152016-01-14 16:56:26 -0800101 private static final int TABLE_FIRST = 0;
102 private static final int TABLE_IN_PORT = 1;
103 private static final int TABLE_ACCESS_TYPE = 2;
104 private static final int TABLE_IN_SERVICE = 3;
105 private static final int TABLE_DST_IP = 4;
106 private static final int TABLE_TUNNEL_IN = 5;
Hyunsun Moon6d247342016-02-12 12:48:47 -0800107 private static final int TABLE_Q_IN_Q = 6;
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800108
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800109 private static final int MANAGEMENT_PRIORITY = 55000;
Hyunsun Moon6d247342016-02-12 12:48:47 -0800110 private static final int VSG_PRIORITY = 55000;
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800111 private static final int HIGH_PRIORITY = 50000;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800112 private static final int DEFAULT_PRIORITY = 5000;
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800113 private static final int LOW_PRIORITY = 4000;
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800114 private static final int LOWEST_PRIORITY = 0;
Hyunsun Moon4a915152016-01-14 16:56:26 -0800115
116 private static final int VXLAN_UDP_PORT = 4789;
Hyunsun Moon6d247342016-02-12 12:48:47 -0800117 private static final VlanId VLAN_WAN = VlanId.vlanId((short) 500);
118
119 private static final String PORT_NAME = "portName";
120 private static final String DATA_PLANE_INTF = "dataPlaneIntf";
121 private static final String S_TAG = "stag";
Hyunsun Moon98025542016-03-08 04:36:02 -0800122 private static final String OVS_HW_VERSION = "Open vSwitch";
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800123
124 private final ApplicationId appId;
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800125 private final FlowRuleService flowRuleService;
126 private final DeviceService deviceService;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800127 private final DriverService driverService;
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800128 private final GroupService groupService;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800129 private final String tunnelType;
130
131 /**
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800132 * Creates a new rule populator.
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800133 *
134 * @param appId application id
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800135 * @param flowRuleService flow rule service
136 * @param deviceService device service
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800137 * @param driverService driver service
Jian Lidfba7392016-01-22 16:46:58 -0800138 * @param groupService group service
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800139 * @param tunnelType tunnel type
140 */
141 public CordVtnRuleInstaller(ApplicationId appId,
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800142 FlowRuleService flowRuleService,
143 DeviceService deviceService,
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800144 DriverService driverService,
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800145 GroupService groupService,
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800146 String tunnelType) {
147 this.appId = appId;
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800148 this.flowRuleService = flowRuleService;
149 this.deviceService = deviceService;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800150 this.driverService = driverService;
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800151 this.groupService = groupService;
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800152 this.tunnelType = checkNotNull(tunnelType);
153 }
154
155 /**
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800156 * Installs table miss rule to a give device.
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800157 *
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800158 * @param deviceId device id to install the rules
Hyunsun Moon133fd792016-02-09 01:55:48 -0800159 * @param dpIntf data plane interface name
160 * @param dpIp data plane ip address
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800161 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800162 public void init(DeviceId deviceId, String dpIntf, IpAddress dpIp) {
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800163 // default is drop packets which can be accomplished without
164 // a table miss entry for all table.
Hyunsun Moon4a915152016-01-14 16:56:26 -0800165 PortNumber tunnelPort = getTunnelPort(deviceId);
Hyunsun Moon133fd792016-02-09 01:55:48 -0800166 PortNumber dpPort = getDpPort(deviceId, dpIntf);
Hyunsun Moon4a915152016-01-14 16:56:26 -0800167
Hyunsun Moon133fd792016-02-09 01:55:48 -0800168 processFirstTable(deviceId, dpPort, dpIp);
169 processInPortTable(deviceId, tunnelPort, dpPort);
170 processAccessTypeTable(deviceId, dpPort);
Hyunsun Moon6d247342016-02-12 12:48:47 -0800171 processQInQTable(deviceId, dpPort);
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800172 }
173
174 /**
Hyunsun Moon2062e7e2016-03-08 15:37:57 -0800175 * Flush flows installed by this application.
176 */
177 public void flushRules() {
178 flowRuleService.getFlowRulesById(appId).forEach(flowRule -> processFlowRule(false, flowRule));
179 }
180
181 /**
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800182 * Populates basic rules that connect a VM to the other VMs in the system.
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800183 *
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800184 * @param host host
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800185 * @param tunnelIp tunnel ip
186 * @param vNet openstack network
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800187 */
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800188 public void populateBasicConnectionRules(Host host, IpAddress tunnelIp, OpenstackNetwork vNet) {
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800189 checkNotNull(host);
190 checkNotNull(vNet);
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800191
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800192 DeviceId deviceId = host.location().deviceId();
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800193 PortNumber inPort = host.location().port();
194 MacAddress dstMac = host.mac();
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800195 IpAddress hostIp = host.ipAddresses().stream().findFirst().get();
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800196 long tunnelId = Long.parseLong(vNet.segmentId());
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800197
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800198 OpenstackSubnet subnet = vNet.subnets().stream()
199 .findFirst()
200 .orElse(null);
201
202 if (subnet == null) {
203 log.error("Failed to get subnet for {}", host.id());
204 return;
205 }
206
207 populateLocalInPortRule(deviceId, inPort, hostIp);
208 populateDirectAccessRule(Ip4Prefix.valueOf(subnet.cidr()), Ip4Prefix.valueOf(subnet.cidr()));
Hyunsun Moon81ab23b2016-02-01 23:00:56 -0800209 populateServiceIsolationRule(Ip4Prefix.valueOf(subnet.cidr()));
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800210 populateDstIpRule(deviceId, inPort, dstMac, hostIp, tunnelId, tunnelIp);
211 populateTunnelInRule(deviceId, inPort, dstMac, tunnelId);
Hyunsun Moon9f0814b2015-11-04 17:34:35 -0800212 }
213
214 /**
Hyunsun Moond5f6c3b2016-02-18 15:37:22 -0800215 * Removes all rules related to a given service VM host.
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800216 *
217 * @param host host to be removed
218 */
219 public void removeBasicConnectionRules(Host host) {
220 checkNotNull(host);
221
222 DeviceId deviceId = host.location().deviceId();
223 MacAddress mac = host.mac();
224 PortNumber port = host.location().port();
225 IpAddress ip = host.ipAddresses().stream().findFirst().orElse(null);
226
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800227 for (FlowRule flowRule : flowRuleService.getFlowRulesById(appId)) {
228 if (flowRule.deviceId().equals(deviceId)) {
229 PortNumber inPort = getInPort(flowRule);
230 if (inPort != null && inPort.equals(port)) {
231 processFlowRule(false, flowRule);
232 continue;
233 }
Hyunsun Moond5f6c3b2016-02-18 15:37:22 -0800234
235 PortNumber output = getOutputFromTreatment(flowRule);
236 if (output != null && output.equals(host.location().port())) {
237 processFlowRule(false, flowRule);
238 }
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800239 }
240
241 MacAddress dstMac = getDstMacFromTreatment(flowRule);
242 if (dstMac != null && dstMac.equals(mac)) {
243 processFlowRule(false, flowRule);
244 continue;
245 }
246
247 dstMac = getDstMacFromSelector(flowRule);
248 if (dstMac != null && dstMac.equals(mac)) {
249 processFlowRule(false, flowRule);
250 continue;
251 }
252
253 IpPrefix dstIp = getDstIpFromSelector(flowRule);
254 if (dstIp != null && dstIp.equals(ip.toIpPrefix())) {
255 processFlowRule(false, flowRule);
256 }
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800257 }
258
259 // TODO uninstall same network access rule in access table if no vm exists in the network
260 }
261
262 /**
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800263 * Populates service dependency rules.
264 *
265 * @param tService tenant cord service
266 * @param pService provider cord service
Hyunsun Moon640f183e2016-02-10 17:02:37 -0800267 * @param isBidirectional true to enable bidirectional connection between two services
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800268 */
Hyunsun Moon640f183e2016-02-10 17:02:37 -0800269 public void populateServiceDependencyRules(CordService tService, CordService pService,
270 boolean isBidirectional) {
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800271 checkNotNull(tService);
272 checkNotNull(pService);
273
274 Ip4Prefix srcRange = tService.serviceIpRange().getIp4Prefix();
275 Ip4Prefix dstRange = pService.serviceIpRange().getIp4Prefix();
276 Ip4Address serviceIp = pService.serviceIp().getIp4Address();
277
278 Map<DeviceId, GroupId> outGroups = Maps.newHashMap();
279 Map<DeviceId, Set<PortNumber>> inPorts = Maps.newHashMap();
280
281 for (Device device : deviceService.getAvailableDevices(SWITCH)) {
Hyunsun Moon98025542016-03-08 04:36:02 -0800282 if (!device.hwVersion().equals(OVS_HW_VERSION)) {
283 continue;
284 }
285
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800286 GroupId groupId = createServiceGroup(device.id(), pService);
287 outGroups.put(device.id(), groupId);
288
289 Set<PortNumber> vms = tService.hosts().keySet()
290 .stream()
291 .filter(host -> host.location().deviceId().equals(device.id()))
292 .map(host -> host.location().port())
293 .collect(Collectors.toSet());
294 inPorts.put(device.id(), vms);
295 }
296
297 populateIndirectAccessRule(srcRange, serviceIp, outGroups);
298 populateDirectAccessRule(srcRange, dstRange);
Hyunsun Moon640f183e2016-02-10 17:02:37 -0800299 if (isBidirectional) {
300 populateDirectAccessRule(dstRange, srcRange);
301 }
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800302 populateInServiceRule(inPorts, outGroups);
303 }
304
305 /**
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800306 * Removes service dependency rules.
307 *
308 * @param tService tenant cord service
309 * @param pService provider cord service
310 */
311 public void removeServiceDependencyRules(CordService tService, CordService pService) {
312 checkNotNull(tService);
313 checkNotNull(pService);
314
315 Ip4Prefix srcRange = tService.serviceIpRange().getIp4Prefix();
316 Ip4Prefix dstRange = pService.serviceIpRange().getIp4Prefix();
317 IpPrefix serviceIp = pService.serviceIp().toIpPrefix();
318
319 Map<DeviceId, GroupId> outGroups = Maps.newHashMap();
320 GroupKey groupKey = new DefaultGroupKey(pService.id().id().getBytes());
321
Hyunsun Moon98025542016-03-08 04:36:02 -0800322 for (Device device : deviceService.getAvailableDevices(SWITCH)) {
323 if (!device.hwVersion().equals(OVS_HW_VERSION)) {
324 continue;
325 }
326
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800327 Group group = groupService.getGroup(device.id(), groupKey);
328 if (group != null) {
329 outGroups.put(device.id(), group.id());
330 }
Hyunsun Moon98025542016-03-08 04:36:02 -0800331 }
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800332
333 for (FlowRule flowRule : flowRuleService.getFlowRulesById(appId)) {
334 IpPrefix dstIp = getDstIpFromSelector(flowRule);
335 IpPrefix srcIp = getSrcIpFromSelector(flowRule);
336
337 if (dstIp != null && dstIp.equals(serviceIp)) {
338 processFlowRule(false, flowRule);
339 continue;
340 }
341
342 if (dstIp != null && srcIp != null) {
343 if (dstIp.equals(dstRange) && srcIp.equals(srcRange)) {
344 processFlowRule(false, flowRule);
345 continue;
346 }
347
348 if (dstIp.equals(srcRange) && srcIp.equals(dstRange)) {
349 processFlowRule(false, flowRule);
350 continue;
351 }
352 }
353
354 GroupId groupId = getGroupIdFromTreatment(flowRule);
355 if (groupId != null && groupId.equals(outGroups.get(flowRule.deviceId()))) {
356 processFlowRule(false, flowRule);
357 }
358 }
359
360 // TODO remove the group if it is not in use
361 }
362
363 /**
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800364 * Updates group buckets for a given service to all devices.
365 *
366 * @param service cord service
367 */
368 public void updateServiceGroup(CordService service) {
369 checkNotNull(service);
370
371 GroupKey groupKey = getGroupKey(service.id());
372
373 for (Device device : deviceService.getAvailableDevices(SWITCH)) {
Hyunsun Moon98025542016-03-08 04:36:02 -0800374 if (!device.hwVersion().equals(OVS_HW_VERSION)) {
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800375 continue;
376 }
377
Hyunsun Moon98025542016-03-08 04:36:02 -0800378 DeviceId deviceId = device.id();
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800379 Group group = groupService.getGroup(deviceId, groupKey);
380 if (group == null) {
Hyunsun Moonb77b60f2016-01-15 20:03:18 -0800381 log.trace("No group exists for service {} in {}, do nothing.", service.id(), deviceId);
Hyunsun Moon4161e6f2016-01-07 01:32:31 -0800382 continue;
383 }
384
385 List<GroupBucket> oldBuckets = group.buckets().buckets();
386 List<GroupBucket> newBuckets = getServiceGroupBuckets(
387 deviceId, service.segmentationId(), service.hosts()).buckets();
388
389 if (oldBuckets.equals(newBuckets)) {
390 continue;
391 }
392
393 List<GroupBucket> bucketsToRemove = new ArrayList<>(oldBuckets);
394 bucketsToRemove.removeAll(newBuckets);
395 if (!bucketsToRemove.isEmpty()) {
396 groupService.removeBucketsFromGroup(
397 deviceId,
398 groupKey,
399 new GroupBuckets(bucketsToRemove),
400 groupKey, appId);
401 }
402
403 List<GroupBucket> bucketsToAdd = new ArrayList<>(newBuckets);
404 bucketsToAdd.removeAll(oldBuckets);
405 if (!bucketsToAdd.isEmpty()) {
406 groupService.addBucketsToGroup(
407 deviceId,
408 groupKey,
409 new GroupBuckets(bucketsToAdd),
410 groupKey, appId);
411 }
412 }
413 }
414
415 /**
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800416 * Populates flow rules for management network access.
417 *
418 * @param host host which has management network interface
419 * @param mService management network service
420 */
421 public void populateManagementNetworkRules(Host host, CordService mService) {
422 checkNotNull(mService);
423
424 DeviceId deviceId = host.location().deviceId();
425 IpAddress hostIp = host.ipAddresses().stream().findFirst().get();
426
427 TrafficSelector selector = DefaultTrafficSelector.builder()
428 .matchEthType(Ethernet.TYPE_ARP)
429 .matchArpTpa(mService.serviceIp().getIp4Address())
430 .build();
431
432 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
433 .setOutput(PortNumber.LOCAL)
434 .build();
435
436 FlowRule flowRule = DefaultFlowRule.builder()
437 .fromApp(appId)
438 .withSelector(selector)
439 .withTreatment(treatment)
440 .withPriority(MANAGEMENT_PRIORITY)
441 .forDevice(deviceId)
442 .forTable(TABLE_FIRST)
443 .makePermanent()
444 .build();
445
446 processFlowRule(true, flowRule);
447
448 selector = DefaultTrafficSelector.builder()
449 .matchInPort(PortNumber.LOCAL)
450 .matchEthType(Ethernet.TYPE_ARP)
451 .matchArpTpa(hostIp.getIp4Address())
452 .build();
453
454 treatment = DefaultTrafficTreatment.builder()
455 .setOutput(host.location().port())
456 .build();
457
458 flowRule = DefaultFlowRule.builder()
459 .fromApp(appId)
460 .withSelector(selector)
461 .withTreatment(treatment)
462 .withPriority(MANAGEMENT_PRIORITY)
463 .forDevice(deviceId)
464 .forTable(TABLE_FIRST)
465 .makePermanent()
466 .build();
467
468 processFlowRule(true, flowRule);
469
470 selector = DefaultTrafficSelector.builder()
471 .matchInPort(PortNumber.LOCAL)
472 .matchEthType(Ethernet.TYPE_IPV4)
473 .matchIPDst(mService.serviceIpRange())
474 .build();
475
476 treatment = DefaultTrafficTreatment.builder()
477 .transition(TABLE_DST_IP)
478 .build();
479
480 flowRule = DefaultFlowRule.builder()
481 .fromApp(appId)
482 .withSelector(selector)
483 .withTreatment(treatment)
484 .withPriority(MANAGEMENT_PRIORITY)
485 .forDevice(deviceId)
486 .forTable(TABLE_FIRST)
487 .makePermanent()
488 .build();
489
490 processFlowRule(true, flowRule);
491
492 selector = DefaultTrafficSelector.builder()
493 .matchEthType(Ethernet.TYPE_IPV4)
494 .matchIPDst(mService.serviceIp().toIpPrefix())
495 .build();
496
497 treatment = DefaultTrafficTreatment.builder()
498 .setOutput(PortNumber.LOCAL)
499 .build();
500
501 flowRule = DefaultFlowRule.builder()
502 .fromApp(appId)
503 .withSelector(selector)
504 .withTreatment(treatment)
505 .withPriority(MANAGEMENT_PRIORITY)
506 .forDevice(deviceId)
507 .forTable(TABLE_ACCESS_TYPE)
508 .makePermanent()
509 .build();
510
511 processFlowRule(true, flowRule);
512 }
513
514 /**
515 * Removes management network access rules.
516 *
517 * @param host host to be removed
518 * @param mService service for management network
519 */
520 public void removeManagementNetworkRules(Host host, CordService mService) {
521 checkNotNull(mService);
Hyunsun Moond5f6c3b2016-02-18 15:37:22 -0800522 // TODO remove management network specific rules
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800523 }
524
525 /**
Hyunsun Moon6d247342016-02-12 12:48:47 -0800526 * Populates rules for vSG VM.
527 *
528 * @param vSgHost vSG host
529 * @param vSgIps set of ip addresses of vSGs running inside the vSG VM
530 */
531 public void populateSubscriberGatewayRules(Host vSgHost, Set<IpAddress> vSgIps) {
532 VlanId serviceVlan = getServiceVlan(vSgHost);
533 PortNumber dpPort = getDpPort(vSgHost);
534
535 if (serviceVlan == null || dpPort == null) {
536 log.warn("Failed to populate rules for vSG VM {}", vSgHost.id());
537 return;
538 }
539
540 // for traffics with s-tag, strip the tag and take through the vSG VM
541 TrafficSelector selector = DefaultTrafficSelector.builder()
Hyunsun Moond5f6c3b2016-02-18 15:37:22 -0800542 .matchInPort(dpPort)
Hyunsun Moon6d247342016-02-12 12:48:47 -0800543 .matchVlanId(serviceVlan)
544 .build();
545
546 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
Hyunsun Moon6d247342016-02-12 12:48:47 -0800547 .setOutput(vSgHost.location().port())
548 .build();
549
550 FlowRule flowRule = DefaultFlowRule.builder()
551 .fromApp(appId)
552 .withSelector(selector)
553 .withTreatment(treatment)
554 .withPriority(DEFAULT_PRIORITY)
555 .forDevice(vSgHost.location().deviceId())
556 .forTable(TABLE_Q_IN_Q)
557 .makePermanent()
558 .build();
559
560 processFlowRule(true, flowRule);
561
562 // for traffics with customer vlan, tag with the service vlan based on input port with
563 // lower priority to avoid conflict with WAN tag
564 selector = DefaultTrafficSelector.builder()
565 .matchInPort(vSgHost.location().port())
Hyunsun Moond5f6c3b2016-02-18 15:37:22 -0800566 .matchVlanId(serviceVlan)
Hyunsun Moon6d247342016-02-12 12:48:47 -0800567 .build();
568
569 treatment = DefaultTrafficTreatment.builder()
Hyunsun Moon6d247342016-02-12 12:48:47 -0800570 .setOutput(dpPort)
571 .build();
572
573 flowRule = DefaultFlowRule.builder()
574 .fromApp(appId)
575 .withSelector(selector)
576 .withTreatment(treatment)
Hyunsun Moond5f6c3b2016-02-18 15:37:22 -0800577 .withPriority(DEFAULT_PRIORITY)
Hyunsun Moon6d247342016-02-12 12:48:47 -0800578 .forDevice(vSgHost.location().deviceId())
579 .forTable(TABLE_Q_IN_Q)
580 .makePermanent()
581 .build();
582
583 processFlowRule(true, flowRule);
584
585 // for traffic coming from WAN, tag 500 and take through the vSG VM
586 // based on destination ip
587 vSgIps.stream().forEach(ip -> {
588 TrafficSelector downstream = DefaultTrafficSelector.builder()
589 .matchEthType(Ethernet.TYPE_IPV4)
590 .matchIPDst(ip.toIpPrefix())
591 .build();
592
593 TrafficTreatment downstreamTreatment = DefaultTrafficTreatment.builder()
594 .pushVlan()
595 .setVlanId(VLAN_WAN)
596 .setEthDst(vSgHost.mac())
597 .setOutput(vSgHost.location().port())
598 .build();
599
600 FlowRule downstreamFlowRule = DefaultFlowRule.builder()
601 .fromApp(appId)
602 .withSelector(downstream)
603 .withTreatment(downstreamTreatment)
604 .withPriority(DEFAULT_PRIORITY)
605 .forDevice(vSgHost.location().deviceId())
606 .forTable(TABLE_DST_IP)
607 .makePermanent()
608 .build();
609
610 processFlowRule(true, downstreamFlowRule);
611 });
612
613 // remove downstream flow rules for the vSG not shown in vSgIps
614 for (FlowRule rule : flowRuleService.getFlowRulesById(appId)) {
615 if (!rule.deviceId().equals(vSgHost.location().deviceId())) {
616 continue;
617 }
618 PortNumber output = getOutputFromTreatment(rule);
619 if (output == null || !output.equals(vSgHost.location().port()) ||
620 !isVlanPushFromTreatment(rule)) {
621 continue;
622 }
623
624 IpPrefix dstIp = getDstIpFromSelector(rule);
625 if (dstIp != null && !vSgIps.contains(dstIp.address())) {
626 processFlowRule(false, rule);
627 }
628 }
629 }
630
631 /**
Hyunsun Moon4a915152016-01-14 16:56:26 -0800632 * Populates default rules on the first table.
Hyunsun Moon133fd792016-02-09 01:55:48 -0800633 * It includes the rules for shuttling vxlan-encapped packets between ovs and
634 * linux stack,and external network connectivity.
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800635 *
636 * @param deviceId device id
Hyunsun Moon133fd792016-02-09 01:55:48 -0800637 * @param dpPort data plane interface port number
638 * @param dpIp data plane ip address
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800639 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800640 private void processFirstTable(DeviceId deviceId, PortNumber dpPort, IpAddress dpIp) {
Hyunsun Moon4a915152016-01-14 16:56:26 -0800641 // take vxlan packet out onto the physical port
642 TrafficSelector selector = DefaultTrafficSelector.builder()
643 .matchInPort(PortNumber.LOCAL)
644 .build();
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800645
Hyunsun Moon4a915152016-01-14 16:56:26 -0800646 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
Hyunsun Moon133fd792016-02-09 01:55:48 -0800647 .setOutput(dpPort)
Hyunsun Moon4a915152016-01-14 16:56:26 -0800648 .build();
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800649
Hyunsun Moon4a915152016-01-14 16:56:26 -0800650 FlowRule flowRule = DefaultFlowRule.builder()
651 .fromApp(appId)
652 .withSelector(selector)
653 .withTreatment(treatment)
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800654 .withPriority(HIGH_PRIORITY)
Hyunsun Moon4a915152016-01-14 16:56:26 -0800655 .forDevice(deviceId)
656 .forTable(TABLE_FIRST)
657 .makePermanent()
658 .build();
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800659
Hyunsun Moon4a915152016-01-14 16:56:26 -0800660 processFlowRule(true, flowRule);
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800661
Hyunsun Moon4a915152016-01-14 16:56:26 -0800662 // take a vxlan encap'd packet through the Linux stack
663 selector = DefaultTrafficSelector.builder()
Hyunsun Moon133fd792016-02-09 01:55:48 -0800664 .matchInPort(dpPort)
Hyunsun Moon4a915152016-01-14 16:56:26 -0800665 .matchEthType(Ethernet.TYPE_IPV4)
666 .matchIPProtocol(IPv4.PROTOCOL_UDP)
667 .matchUdpDst(TpPort.tpPort(VXLAN_UDP_PORT))
668 .build();
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800669
Hyunsun Moon4a915152016-01-14 16:56:26 -0800670 treatment = DefaultTrafficTreatment.builder()
671 .setOutput(PortNumber.LOCAL)
672 .build();
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800673
Hyunsun Moon4a915152016-01-14 16:56:26 -0800674 flowRule = DefaultFlowRule.builder()
675 .fromApp(appId)
676 .withSelector(selector)
677 .withTreatment(treatment)
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800678 .withPriority(HIGH_PRIORITY)
Hyunsun Moon4a915152016-01-14 16:56:26 -0800679 .forDevice(deviceId)
680 .forTable(TABLE_FIRST)
681 .makePermanent()
682 .build();
683
684 processFlowRule(true, flowRule);
685
Hyunsun Moon133fd792016-02-09 01:55:48 -0800686 // take a packet to the data plane ip through Linux stack
Hyunsun Moonb4f85082016-01-15 20:11:41 -0800687 selector = DefaultTrafficSelector.builder()
Hyunsun Moon133fd792016-02-09 01:55:48 -0800688 .matchInPort(dpPort)
Hyunsun Moonb4f85082016-01-15 20:11:41 -0800689 .matchEthType(Ethernet.TYPE_IPV4)
Hyunsun Moon133fd792016-02-09 01:55:48 -0800690 .matchIPDst(dpIp.toIpPrefix())
Hyunsun Moonb4f85082016-01-15 20:11:41 -0800691 .build();
692
693 treatment = DefaultTrafficTreatment.builder()
694 .setOutput(PortNumber.LOCAL)
695 .build();
696
697 flowRule = DefaultFlowRule.builder()
698 .fromApp(appId)
699 .withSelector(selector)
700 .withTreatment(treatment)
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800701 .withPriority(HIGH_PRIORITY)
Hyunsun Moonb4f85082016-01-15 20:11:41 -0800702 .forDevice(deviceId)
703 .forTable(TABLE_FIRST)
704 .makePermanent()
705 .build();
706
707 processFlowRule(true, flowRule);
708
709 // take an arp packet from physical through Linux stack
710 selector = DefaultTrafficSelector.builder()
Hyunsun Moon133fd792016-02-09 01:55:48 -0800711 .matchInPort(dpPort)
Hyunsun Moonb4f85082016-01-15 20:11:41 -0800712 .matchEthType(Ethernet.TYPE_ARP)
Hyunsun Moon6d247342016-02-12 12:48:47 -0800713 .matchArpTpa(dpIp.getIp4Address())
Hyunsun Moonb4f85082016-01-15 20:11:41 -0800714 .build();
715
716 treatment = DefaultTrafficTreatment.builder()
717 .setOutput(PortNumber.LOCAL)
718 .build();
719
720 flowRule = DefaultFlowRule.builder()
721 .fromApp(appId)
722 .withSelector(selector)
723 .withTreatment(treatment)
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800724 .withPriority(HIGH_PRIORITY)
Hyunsun Moonb4f85082016-01-15 20:11:41 -0800725 .forDevice(deviceId)
726 .forTable(TABLE_FIRST)
727 .makePermanent()
728 .build();
729
730 processFlowRule(true, flowRule);
731
Hyunsun Moon4a915152016-01-14 16:56:26 -0800732 // take all else to the next table
733 selector = DefaultTrafficSelector.builder()
734 .build();
735
736 treatment = DefaultTrafficTreatment.builder()
737 .transition(TABLE_IN_PORT)
738 .build();
739
740 flowRule = DefaultFlowRule.builder()
741 .fromApp(appId)
742 .withSelector(selector)
743 .withTreatment(treatment)
744 .withPriority(LOWEST_PRIORITY)
745 .forDevice(deviceId)
746 .forTable(TABLE_FIRST)
747 .makePermanent()
748 .build();
749
750 processFlowRule(true, flowRule);
Hyunsun Moon6d247342016-02-12 12:48:47 -0800751
752 // take all vlan tagged packet to the Q_IN_Q table
753 selector = DefaultTrafficSelector.builder()
754 .matchVlanId(VlanId.ANY)
755 .build();
756
757 treatment = DefaultTrafficTreatment.builder()
758 .transition(TABLE_Q_IN_Q)
759 .build();
760
761 flowRule = DefaultFlowRule.builder()
762 .fromApp(appId)
763 .withSelector(selector)
764 .withTreatment(treatment)
765 .withPriority(VSG_PRIORITY)
766 .forDevice(deviceId)
767 .forTable(TABLE_FIRST)
768 .makePermanent()
769 .build();
770
771 processFlowRule(true, flowRule);
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800772 }
773
774 /**
Hyunsun Moon133fd792016-02-09 01:55:48 -0800775 * Forward table miss packets in ACCESS_TYPE table to data plane port.
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800776 *
777 * @param deviceId device id
Hyunsun Moon133fd792016-02-09 01:55:48 -0800778 * @param dpPort data plane interface port number
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800779 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800780 private void processAccessTypeTable(DeviceId deviceId, PortNumber dpPort) {
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800781 TrafficSelector selector = DefaultTrafficSelector.builder()
782 .build();
783
784 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
Hyunsun Moon133fd792016-02-09 01:55:48 -0800785 .setOutput(dpPort)
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800786 .build();
787
788 FlowRule flowRule = DefaultFlowRule.builder()
789 .fromApp(appId)
790 .withSelector(selector)
791 .withTreatment(treatment)
792 .withPriority(LOWEST_PRIORITY)
793 .forDevice(deviceId)
794 .forTable(TABLE_ACCESS_TYPE)
795 .makePermanent()
796 .build();
797
798 processFlowRule(true, flowRule);
799 }
800
801 /**
Hyunsun Moon4a915152016-01-14 16:56:26 -0800802 * Populates default rules for IN_PORT table.
803 * All packets from tunnel port are forwarded to TUNNEL_ID table and all packets
Hyunsun Moon133fd792016-02-09 01:55:48 -0800804 * from data plane interface port to ACCESS_TYPE table.
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800805 *
806 * @param deviceId device id to install the rules
Hyunsun Moon4a915152016-01-14 16:56:26 -0800807 * @param tunnelPort tunnel port number
Hyunsun Moon133fd792016-02-09 01:55:48 -0800808 * @param dpPort data plane interface port number
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800809 */
Hyunsun Moon133fd792016-02-09 01:55:48 -0800810 private void processInPortTable(DeviceId deviceId, PortNumber tunnelPort, PortNumber dpPort) {
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800811 checkNotNull(tunnelPort);
812
813 TrafficSelector selector = DefaultTrafficSelector.builder()
814 .matchInPort(tunnelPort)
815 .build();
Hyunsun Moon4a915152016-01-14 16:56:26 -0800816
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800817 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
818 .transition(TABLE_TUNNEL_IN)
819 .build();
820
821 FlowRule flowRule = DefaultFlowRule.builder()
822 .fromApp(appId)
823 .withSelector(selector)
824 .withTreatment(treatment)
825 .withPriority(DEFAULT_PRIORITY)
826 .forDevice(deviceId)
827 .forTable(TABLE_IN_PORT)
828 .makePermanent()
829 .build();
830
831 processFlowRule(true, flowRule);
Hyunsun Moon4a915152016-01-14 16:56:26 -0800832
833 selector = DefaultTrafficSelector.builder()
Hyunsun Moon133fd792016-02-09 01:55:48 -0800834 .matchInPort(dpPort)
Hyunsun Moon4a915152016-01-14 16:56:26 -0800835 .build();
836
837 treatment = DefaultTrafficTreatment.builder()
838 .transition(TABLE_DST_IP)
839 .build();
840
841 flowRule = DefaultFlowRule.builder()
842 .fromApp(appId)
843 .withSelector(selector)
844 .withTreatment(treatment)
845 .withPriority(DEFAULT_PRIORITY)
846 .forDevice(deviceId)
847 .forTable(TABLE_IN_PORT)
848 .makePermanent()
849 .build();
850
851 processFlowRule(true, flowRule);
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800852 }
853
854 /**
Hyunsun Moon6d247342016-02-12 12:48:47 -0800855 * Populates default rules for Q_IN_Q table.
856 *
857 * @param deviceId device id
858 * @param dpPort data plane interface port number
859 */
860 private void processQInQTable(DeviceId deviceId, PortNumber dpPort) {
861 // for traffic going out to WAN, strip vid 500 and take through data plane interface
862 TrafficSelector selector = DefaultTrafficSelector.builder()
863 .matchVlanId(VLAN_WAN)
864 .build();
865
866 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
867 .popVlan()
868 .setOutput(dpPort)
869 .build();
870
871 FlowRule flowRule = DefaultFlowRule.builder()
872 .fromApp(appId)
873 .withSelector(selector)
874 .withTreatment(treatment)
875 .withPriority(DEFAULT_PRIORITY)
876 .forDevice(deviceId)
877 .forTable(TABLE_Q_IN_Q)
878 .makePermanent()
879 .build();
880
881 processFlowRule(true, flowRule);
882
883 selector = DefaultTrafficSelector.builder()
884 .matchVlanId(VLAN_WAN)
885 .matchEthType(Ethernet.TYPE_ARP)
886 .build();
887
888 treatment = DefaultTrafficTreatment.builder()
889 .setOutput(PortNumber.CONTROLLER)
890 .build();
891
892 flowRule = DefaultFlowRule.builder()
893 .fromApp(appId)
894 .withSelector(selector)
895 .withTreatment(treatment)
896 .withPriority(HIGH_PRIORITY)
897 .forDevice(deviceId)
898 .forTable(TABLE_Q_IN_Q)
899 .makePermanent()
900 .build();
901
902 processFlowRule(true, flowRule);
903 }
904
905 /**
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800906 * Populates rules for local in port in IN_PORT table.
907 * Flows from a given in port, whose source IP is service IP transition
908 * to DST_TYPE table. Other flows transition to IN_SERVICE table.
909 *
910 * @param deviceId device id to install the rules
911 * @param inPort in port
912 * @param srcIp source ip
913 */
914 private void populateLocalInPortRule(DeviceId deviceId, PortNumber inPort, IpAddress srcIp) {
915 TrafficSelector selector = DefaultTrafficSelector.builder()
916 .matchInPort(inPort)
917 .matchEthType(Ethernet.TYPE_IPV4)
918 .matchIPSrc(srcIp.toIpPrefix())
919 .build();
Hyunsun Moon4a915152016-01-14 16:56:26 -0800920
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800921 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
922 .transition(TABLE_ACCESS_TYPE)
923 .build();
924
925
926 FlowRule flowRule = DefaultFlowRule.builder()
927 .fromApp(appId)
928 .withSelector(selector)
929 .withTreatment(treatment)
930 .withPriority(DEFAULT_PRIORITY)
931 .forDevice(deviceId)
932 .forTable(TABLE_IN_PORT)
933 .makePermanent()
934 .build();
935
936 processFlowRule(true, flowRule);
937
938 selector = DefaultTrafficSelector.builder()
939 .matchInPort(inPort)
940 .build();
Hyunsun Moon4a915152016-01-14 16:56:26 -0800941
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800942 treatment = DefaultTrafficTreatment.builder()
943 .transition(TABLE_IN_SERVICE)
944 .build();
945
946 flowRule = DefaultFlowRule.builder()
947 .fromApp(appId)
948 .withSelector(selector)
949 .withTreatment(treatment)
Hyunsun Moond52bffc2016-01-29 18:57:05 -0800950 .withPriority(LOW_PRIORITY)
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800951 .forDevice(deviceId)
952 .forTable(TABLE_IN_PORT)
953 .makePermanent()
954 .build();
955
956 processFlowRule(true, flowRule);
957 }
958
959 /**
960 * Populates direct VM access rules for ACCESS_TYPE table.
961 * These rules are installed to all devices.
962 *
963 * @param srcRange source ip range
964 * @param dstRange destination ip range
965 */
966 private void populateDirectAccessRule(Ip4Prefix srcRange, Ip4Prefix dstRange) {
967 TrafficSelector selector = DefaultTrafficSelector.builder()
968 .matchEthType(Ethernet.TYPE_IPV4)
969 .matchIPSrc(srcRange)
970 .matchIPDst(dstRange)
971 .build();
Hyunsun Moon4a915152016-01-14 16:56:26 -0800972
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800973 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
974 .transition(TABLE_DST_IP)
975 .build();
976
977 for (Device device : deviceService.getAvailableDevices(SWITCH)) {
Hyunsun Moon98025542016-03-08 04:36:02 -0800978 if (!device.hwVersion().equals(OVS_HW_VERSION)) {
979 continue;
980 }
981
Hyunsun Moonc71231d2015-12-16 20:53:23 -0800982 FlowRule flowRuleDirect = DefaultFlowRule.builder()
983 .fromApp(appId)
984 .withSelector(selector)
985 .withTreatment(treatment)
Hyunsun Moon81ab23b2016-02-01 23:00:56 -0800986 .withPriority(DEFAULT_PRIORITY)
987 .forDevice(device.id())
988 .forTable(TABLE_ACCESS_TYPE)
989 .makePermanent()
990 .build();
991
992 processFlowRule(true, flowRuleDirect);
993 }
994 }
995
996 /**
997 * Populates drop rules that does not match any direct access rules but has
998 * destination to a different service network in ACCESS_TYPE table.
999 *
1000 * @param dstRange destination ip range
1001 */
1002 private void populateServiceIsolationRule(Ip4Prefix dstRange) {
1003 TrafficSelector selector = DefaultTrafficSelector.builder()
1004 .matchEthType(Ethernet.TYPE_IPV4)
1005 .matchIPDst(dstRange)
1006 .build();
1007
1008 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
1009 .drop()
1010 .build();
1011
1012 for (Device device : deviceService.getAvailableDevices(SWITCH)) {
Hyunsun Moon98025542016-03-08 04:36:02 -08001013 if (!device.hwVersion().equals(OVS_HW_VERSION)) {
1014 continue;
1015 }
1016
Hyunsun Moon81ab23b2016-02-01 23:00:56 -08001017 FlowRule flowRuleDirect = DefaultFlowRule.builder()
1018 .fromApp(appId)
1019 .withSelector(selector)
1020 .withTreatment(treatment)
Hyunsun Moond52bffc2016-01-29 18:57:05 -08001021 .withPriority(LOW_PRIORITY)
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001022 .forDevice(device.id())
1023 .forTable(TABLE_ACCESS_TYPE)
1024 .makePermanent()
1025 .build();
1026
1027 processFlowRule(true, flowRuleDirect);
1028 }
1029 }
1030
1031 /**
1032 * Populates indirect service access rules for ACCESS_TYPE table.
1033 * These rules are installed to all devices.
1034 *
1035 * @param srcRange source range
1036 * @param serviceIp service ip
1037 * @param outGroups list of output group
1038 */
1039 private void populateIndirectAccessRule(Ip4Prefix srcRange, Ip4Address serviceIp,
1040 Map<DeviceId, GroupId> outGroups) {
1041 TrafficSelector selector = DefaultTrafficSelector.builder()
1042 .matchEthType(Ethernet.TYPE_IPV4)
1043 .matchIPSrc(srcRange)
1044 .matchIPDst(serviceIp.toIpPrefix())
1045 .build();
1046
1047 for (Map.Entry<DeviceId, GroupId> outGroup : outGroups.entrySet()) {
1048 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
1049 .group(outGroup.getValue())
1050 .build();
1051
1052 FlowRule flowRule = DefaultFlowRule.builder()
1053 .fromApp(appId)
1054 .withSelector(selector)
1055 .withTreatment(treatment)
Hyunsun Moon81ab23b2016-02-01 23:00:56 -08001056 .withPriority(HIGH_PRIORITY)
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001057 .forDevice(outGroup.getKey())
1058 .forTable(TABLE_ACCESS_TYPE)
1059 .makePermanent()
1060 .build();
1061
1062 processFlowRule(true, flowRule);
1063 }
1064 }
1065
1066 /**
1067 * Populates flow rules for IN_SERVICE table.
1068 *
1069 * @param inPorts list of inports related to the service for each device
1070 * @param outGroups set of output groups
1071 */
1072 private void populateInServiceRule(Map<DeviceId, Set<PortNumber>> inPorts, Map<DeviceId, GroupId> outGroups) {
1073 checkNotNull(inPorts);
1074 checkNotNull(outGroups);
1075
1076 for (Map.Entry<DeviceId, Set<PortNumber>> entry : inPorts.entrySet()) {
1077 Set<PortNumber> ports = entry.getValue();
1078 DeviceId deviceId = entry.getKey();
1079
1080 GroupId groupId = outGroups.get(deviceId);
1081 if (groupId == null) {
1082 continue;
1083 }
1084
1085 ports.stream().forEach(port -> {
1086 TrafficSelector selector = DefaultTrafficSelector.builder()
1087 .matchInPort(port)
1088 .build();
Hyunsun Moon4a915152016-01-14 16:56:26 -08001089
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001090 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
1091 .group(groupId)
1092 .build();
1093
1094 FlowRule flowRule = DefaultFlowRule.builder()
1095 .fromApp(appId)
1096 .withSelector(selector)
1097 .withTreatment(treatment)
1098 .withPriority(DEFAULT_PRIORITY)
1099 .forDevice(deviceId)
1100 .forTable(TABLE_IN_SERVICE)
1101 .makePermanent()
1102 .build();
1103
1104 processFlowRule(true, flowRule);
1105 });
1106 }
1107 }
1108
1109 /**
1110 * Populates flow rules for DST_IP table.
1111 *
1112 * @param deviceId device id
1113 * @param inPort in port
1114 * @param dstMac mac address
1115 * @param dstIp destination ip
1116 * @param tunnelId tunnel id
1117 * @param tunnelIp tunnel remote ip
1118 */
1119 private void populateDstIpRule(DeviceId deviceId, PortNumber inPort, MacAddress dstMac,
1120 IpAddress dstIp, long tunnelId, IpAddress tunnelIp) {
1121 TrafficSelector selector = DefaultTrafficSelector.builder()
1122 .matchEthType(Ethernet.TYPE_IPV4)
1123 .matchIPDst(dstIp.toIpPrefix())
1124 .build();
Hyunsun Moon4a915152016-01-14 16:56:26 -08001125
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001126 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
1127 .setEthDst(dstMac)
1128 .setOutput(inPort)
1129 .build();
1130
1131 FlowRule flowRule = DefaultFlowRule.builder()
1132 .fromApp(appId)
1133 .withSelector(selector)
1134 .withTreatment(treatment)
1135 .withPriority(DEFAULT_PRIORITY)
1136 .forDevice(deviceId)
1137 .forTable(TABLE_DST_IP)
1138 .makePermanent()
1139 .build();
1140
1141 processFlowRule(true, flowRule);
1142
1143 for (Device device : deviceService.getAvailableDevices(SWITCH)) {
Hyunsun Moon98025542016-03-08 04:36:02 -08001144 if (!device.hwVersion().equals(OVS_HW_VERSION)) {
1145 continue;
1146 }
1147
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001148 if (device.id().equals(deviceId)) {
1149 continue;
1150 }
1151
1152 ExtensionTreatment tunnelDst = getTunnelDst(device.id(), tunnelIp.getIp4Address());
1153 if (tunnelDst == null) {
1154 continue;
1155 }
1156
1157 treatment = DefaultTrafficTreatment.builder()
1158 .setEthDst(dstMac)
1159 .setTunnelId(tunnelId)
1160 .extension(tunnelDst, device.id())
1161 .setOutput(getTunnelPort(device.id()))
1162 .build();
1163
1164 flowRule = DefaultFlowRule.builder()
1165 .fromApp(appId)
1166 .withSelector(selector)
1167 .withTreatment(treatment)
1168 .withPriority(DEFAULT_PRIORITY)
1169 .forDevice(device.id())
1170 .forTable(TABLE_DST_IP)
1171 .makePermanent()
1172 .build();
1173
1174 processFlowRule(true, flowRule);
1175 }
1176 }
1177
1178 /**
1179 * Populates flow rules for TUNNEL_ID table.
1180 *
1181 * @param deviceId device id
1182 * @param inPort in port
1183 * @param mac mac address
1184 * @param tunnelId tunnel id
1185 */
1186 private void populateTunnelInRule(DeviceId deviceId, PortNumber inPort, MacAddress mac, long tunnelId) {
1187 TrafficSelector selector = DefaultTrafficSelector.builder()
1188 .matchTunnelId(tunnelId)
1189 .matchEthDst(mac)
1190 .build();
1191
1192 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
1193 .setOutput(inPort)
1194 .build();
1195
1196 FlowRule flowRule = DefaultFlowRule.builder()
1197 .fromApp(appId)
1198 .withSelector(selector)
1199 .withTreatment(treatment)
1200 .withPriority(DEFAULT_PRIORITY)
1201 .forDevice(deviceId)
1202 .forTable(TABLE_TUNNEL_IN)
1203 .makePermanent()
1204 .build();
1205
1206 processFlowRule(true, flowRule);
1207 }
1208
1209 /**
1210 * Installs or uninstall a given rule.
1211 *
1212 * @param install true to install, false to uninstall
1213 * @param rule rule
1214 */
1215 private void processFlowRule(boolean install, FlowRule rule) {
1216 FlowRuleOperations.Builder oBuilder = FlowRuleOperations.builder();
1217 oBuilder = install ? oBuilder.add(rule) : oBuilder.remove(rule);
1218
1219 flowRuleService.apply(oBuilder.build(new FlowRuleOperationsContext() {
1220 @Override
1221 public void onError(FlowRuleOperations ops) {
1222 log.error(String.format("Failed %s, %s", ops.toString(), rule.toString()));
1223 }
1224 }));
1225 }
1226
1227 /**
1228 * Returns tunnel port of the device.
1229 *
1230 * @param deviceId device id
1231 * @return tunnel port number, or null if no tunnel port exists on a given device
1232 */
1233 private PortNumber getTunnelPort(DeviceId deviceId) {
Hyunsun Moon5a4346f2016-01-15 11:41:14 -08001234 Port port = deviceService.getPorts(deviceId).stream()
Hyunsun Moon6d247342016-02-12 12:48:47 -08001235 .filter(p -> p.annotations().value(PORT_NAME).contains(tunnelType))
1236 .findFirst().orElse(null);
Hyunsun Moon5a4346f2016-01-15 11:41:14 -08001237
1238 return port == null ? null : port.number();
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001239 }
1240
1241 /**
Hyunsun Moon133fd792016-02-09 01:55:48 -08001242 * Returns data plane interface port name of a given device.
Hyunsun Moon4a915152016-01-14 16:56:26 -08001243 *
1244 * @param deviceId device id
Hyunsun Moon133fd792016-02-09 01:55:48 -08001245 * @param dpIntf data plane interface port name
1246 * @return data plane interface port number, or null if no such port exists
Hyunsun Moon4a915152016-01-14 16:56:26 -08001247 */
Hyunsun Moon133fd792016-02-09 01:55:48 -08001248 private PortNumber getDpPort(DeviceId deviceId, String dpIntf) {
Hyunsun Moon5a4346f2016-01-15 11:41:14 -08001249 Port port = deviceService.getPorts(deviceId).stream()
Hyunsun Moon6d247342016-02-12 12:48:47 -08001250 .filter(p -> p.annotations().value(PORT_NAME).contains(dpIntf) &&
1251 p.isEnabled())
1252 .findFirst().orElse(null);
Hyunsun Moon5a4346f2016-01-15 11:41:14 -08001253
1254 return port == null ? null : port.number();
Hyunsun Moon4a915152016-01-14 16:56:26 -08001255 }
1256
Hyunsun Moon6d247342016-02-12 12:48:47 -08001257 /** Returns data plane interface port number of a given host.
1258 *
1259 * @param host host
1260 * @return port number, or null
1261 */
1262 private PortNumber getDpPort(Host host) {
1263 String portName = host.annotations().value(DATA_PLANE_INTF);
1264 return portName == null ? null : getDpPort(host.location().deviceId(), portName);
1265 }
1266
1267 /**
1268 * Returns service vlan from a given host.
1269 *
1270 * @param host host
1271 * @return vlan id, or null
1272 */
1273 private VlanId getServiceVlan(Host host) {
1274 String serviceVlan = host.annotations().value(S_TAG);
1275 return serviceVlan == null ? null : VlanId.vlanId(Short.parseShort(serviceVlan));
1276 }
1277
Hyunsun Moon4a915152016-01-14 16:56:26 -08001278 /**
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001279 * Returns the inport from a given flow rule if the rule contains the match of it.
1280 *
1281 * @param flowRule flow rule
1282 * @return port number, or null if the rule doesn't have inport match
1283 */
1284 private PortNumber getInPort(FlowRule flowRule) {
1285 Criterion criterion = flowRule.selector().getCriterion(IN_PORT);
1286 if (criterion != null && criterion instanceof PortCriterion) {
1287 PortCriterion port = (PortCriterion) criterion;
1288 return port.port();
1289 } else {
1290 return null;
1291 }
1292 }
1293
1294 /**
1295 * Returns the destination mac address from a given flow rule if the rule
1296 * contains the instruction of it.
1297 *
1298 * @param flowRule flow rule
1299 * @return mac address, or null if the rule doesn't have destination mac instruction
1300 */
1301 private MacAddress getDstMacFromTreatment(FlowRule flowRule) {
1302 Instruction instruction = flowRule.treatment().allInstructions().stream()
1303 .filter(inst -> inst instanceof ModEtherInstruction &&
1304 ((ModEtherInstruction) inst).subtype().equals(ETH_DST))
1305 .findFirst()
1306 .orElse(null);
1307
1308 if (instruction == null) {
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001309 return null;
1310 }
1311
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001312 return ((ModEtherInstruction) instruction).mac();
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001313 }
1314
1315 /**
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001316 * Returns the destination mac address from a given flow rule if the rule
1317 * contains the match of it.
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001318 *
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001319 * @param flowRule flow rule
1320 * @return mac address, or null if the rule doesn't have destination mac match
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001321 */
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001322 private MacAddress getDstMacFromSelector(FlowRule flowRule) {
1323 Criterion criterion = flowRule.selector().getCriterion(Criterion.Type.ETH_DST);
1324 if (criterion != null && criterion instanceof EthCriterion) {
1325 EthCriterion eth = (EthCriterion) criterion;
1326 return eth.mac();
1327 } else {
1328 return null;
1329 }
1330 }
1331
1332 /**
1333 * Returns the destination IP from a given flow rule if the rule contains
1334 * the match of it.
1335 *
1336 * @param flowRule flow rule
1337 * @return ip prefix, or null if the rule doesn't have ip match
1338 */
1339 private IpPrefix getDstIpFromSelector(FlowRule flowRule) {
1340 Criterion criterion = flowRule.selector().getCriterion(IPV4_DST);
1341 if (criterion != null && criterion instanceof IPCriterion) {
1342 IPCriterion ip = (IPCriterion) criterion;
1343 return ip.ip();
1344 } else {
1345 return null;
1346 }
1347 }
1348
1349 /**
1350 * Returns the source IP from a given flow rule if the rule contains
1351 * the match of it.
1352 *
1353 * @param flowRule flow rule
1354 * @return ip prefix, or null if the rule doesn't have ip match
1355 */
1356 private IpPrefix getSrcIpFromSelector(FlowRule flowRule) {
1357 Criterion criterion = flowRule.selector().getCriterion(IPV4_SRC);
1358 if (criterion != null && criterion instanceof IPCriterion) {
1359 IPCriterion ip = (IPCriterion) criterion;
1360 return ip.ip();
1361 } else {
1362 return null;
1363 }
1364 }
1365
1366 /**
1367 * Returns the group ID from a given flow rule if the rule contains the
1368 * treatment of it.
1369 *
1370 * @param flowRule flow rule
1371 * @return group id, or null if the rule doesn't have group instruction
1372 */
1373 private GroupId getGroupIdFromTreatment(FlowRule flowRule) {
1374 Instruction instruction = flowRule.treatment().allInstructions().stream()
1375 .filter(inst -> inst instanceof Instructions.GroupInstruction)
1376 .findFirst()
1377 .orElse(null);
1378
1379 if (instruction == null) {
1380 return null;
1381 }
1382
1383 return ((Instructions.GroupInstruction) instruction).groupId();
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001384 }
1385
1386 /**
Hyunsun Moond52bffc2016-01-29 18:57:05 -08001387 * Returns the output port number from a given flow rule.
1388 *
1389 * @param flowRule flow rule
1390 * @return port number, or null if the rule does not have output instruction
1391 */
1392 private PortNumber getOutputFromTreatment(FlowRule flowRule) {
1393 Instruction instruction = flowRule.treatment().allInstructions().stream()
Hyunsun Moon6d247342016-02-12 12:48:47 -08001394 .filter(inst -> inst instanceof Instructions.OutputInstruction)
Hyunsun Moond52bffc2016-01-29 18:57:05 -08001395 .findFirst()
1396 .orElse(null);
1397
1398 if (instruction == null) {
1399 return null;
1400 }
1401
1402 return ((Instructions.OutputInstruction) instruction).port();
1403 }
1404
1405 /**
Hyunsun Moon6d247342016-02-12 12:48:47 -08001406 * Returns if a given flow rule has vlan push instruction or not.
1407 *
1408 * @param flowRule flow rule
1409 * @return true if it includes vlan push, or false
1410 */
1411 private boolean isVlanPushFromTreatment(FlowRule flowRule) {
1412 Instruction instruction = flowRule.treatment().allInstructions().stream()
1413 .filter(inst -> inst instanceof L2ModificationInstruction)
1414 .filter(inst -> ((L2ModificationInstruction) inst).subtype().equals(VLAN_PUSH))
1415 .findAny()
1416 .orElse(null);
1417
1418 return instruction != null;
1419 }
1420
1421 /**
Hyunsun Moon4a915152016-01-14 16:56:26 -08001422 * Creates a new group for a given service.
1423 *
1424 * @param deviceId device id to create a group
1425 * @param service cord service
1426 * @return group id, or null if it fails to create
1427 */
1428 private GroupId createServiceGroup(DeviceId deviceId, CordService service) {
1429 checkNotNull(service);
1430
1431 GroupKey groupKey = getGroupKey(service.id());
1432 Group group = groupService.getGroup(deviceId, groupKey);
1433 GroupId groupId = getGroupId(service.id(), deviceId);
1434
1435 if (group != null) {
1436 log.debug("Group {} is already exist in {}", service.id(), deviceId);
1437 return groupId;
1438 }
1439
1440 GroupBuckets buckets = getServiceGroupBuckets(deviceId, service.segmentationId(), service.hosts());
1441 GroupDescription groupDescription = new DefaultGroupDescription(
1442 deviceId,
1443 GroupDescription.Type.SELECT,
1444 buckets,
1445 groupKey,
1446 groupId.id(),
1447 appId);
1448
1449 groupService.addGroup(groupDescription);
1450
1451 return groupId;
1452 }
1453
1454 /**
1455 * Returns group buckets for a given device.
1456 *
1457 * @param deviceId device id
1458 * @param tunnelId tunnel id
1459 * @param hosts list of host
1460 * @return group buckets
1461 */
1462 private GroupBuckets getServiceGroupBuckets(DeviceId deviceId, long tunnelId, Map<Host, IpAddress> hosts) {
1463 List<GroupBucket> buckets = Lists.newArrayList();
1464
1465 for (Map.Entry<Host, IpAddress> entry : hosts.entrySet()) {
1466 Host host = entry.getKey();
1467 Ip4Address remoteIp = entry.getValue().getIp4Address();
1468 DeviceId hostDevice = host.location().deviceId();
1469
1470 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment
1471 .builder()
1472 .setEthDst(host.mac());
1473
1474 if (deviceId.equals(hostDevice)) {
1475 tBuilder.setOutput(host.location().port());
1476 } else {
1477 ExtensionTreatment tunnelDst = getTunnelDst(deviceId, remoteIp);
1478 if (tunnelDst == null) {
1479 continue;
1480 }
1481
1482 tBuilder.extension(tunnelDst, deviceId)
1483 .setTunnelId(tunnelId)
1484 .setOutput(getTunnelPort(hostDevice));
1485 }
1486
1487 buckets.add(DefaultGroupBucket.createSelectGroupBucket(tBuilder.build()));
1488 }
1489
1490 return new GroupBuckets(buckets);
1491 }
1492
1493 /**
1494 * Returns globally unique group ID.
1495 *
1496 * @param serviceId service id
1497 * @param deviceId device id
1498 * @return group id
1499 */
1500 private GroupId getGroupId(CordServiceId serviceId, DeviceId deviceId) {
1501 return new DefaultGroupId(Objects.hash(serviceId, deviceId));
1502 }
1503
1504 /**
1505 * Returns group key of a service.
1506 *
1507 * @param serviceId service id
1508 * @return group key
1509 */
1510 private GroupKey getGroupKey(CordServiceId serviceId) {
1511 return new DefaultGroupKey(serviceId.id().getBytes());
1512 }
1513
1514 /**
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001515 * Returns extension instruction to set tunnel destination.
1516 *
1517 * @param deviceId device id
1518 * @param remoteIp tunnel destination address
1519 * @return extension treatment or null if it fails to get instruction
1520 */
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001521 private ExtensionTreatment getTunnelDst(DeviceId deviceId, Ip4Address remoteIp) {
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001522 try {
1523 Driver driver = driverService.getDriver(deviceId);
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001524 DefaultDriverData driverData = new DefaultDriverData(driver, deviceId);
1525 DriverHandler handler = new DefaultDriverHandler(driverData);
1526 ExtensionTreatmentResolver resolver = handler.behaviour(ExtensionTreatmentResolver.class);
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001527
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001528 ExtensionTreatment treatment =
1529 resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type());
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001530 treatment.setPropertyValue("tunnelDst", remoteIp);
1531
1532 return treatment;
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001533 } catch (ItemNotFoundException | UnsupportedOperationException |
1534 ExtensionPropertyException e) {
1535 log.error("Failed to get extension instruction {}", deviceId);
Hyunsun Moon9f0814b2015-11-04 17:34:35 -08001536 return null;
1537 }
1538 }
1539}
Hyunsun Moonc71231d2015-12-16 20:53:23 -08001540