blob: fb1bafc34c53a6fbc2bcae884a7dea1d7796b50e [file] [log] [blame]
lishuai6c56f5e2015-11-17 16:38:19 +08001/*
Brian O'Connora09fe5b2017-08-03 21:12:30 -07002 * Copyright 2015-present Open Networking Foundation
lishuai6c56f5e2015-11-17 16:38:19 +08003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package org.onosproject.vtn.manager.impl;
17
jiangruic69a7fd2015-11-19 15:40:01 +080018import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST;
lishuai6c56f5e2015-11-17 16:38:19 +080019import static org.slf4j.LoggerFactory.getLogger;
20
Bob zhou59a21062016-05-12 19:40:05 +080021import java.nio.ByteBuffer;
lishuai858efd32015-12-04 14:30:36 +080022import java.util.ArrayList;
lishuaib43dbf72016-01-06 11:11:35 +080023import java.util.Arrays;
lishuai6c56f5e2015-11-17 16:38:19 +080024import java.util.Collection;
25import java.util.HashMap;
26import java.util.HashSet;
27import java.util.Iterator;
jiangruic69a7fd2015-11-19 15:40:01 +080028import java.util.List;
lishuai6c56f5e2015-11-17 16:38:19 +080029import java.util.Map;
30import java.util.Set;
Bob zhou59a21062016-05-12 19:40:05 +080031import java.util.UUID;
lishuai74f2d532015-12-10 17:03:34 +080032import java.util.concurrent.ConcurrentHashMap;
lishuai858efd32015-12-04 14:30:36 +080033import java.util.stream.Collectors;
lishuai6c56f5e2015-11-17 16:38:19 +080034
35import org.apache.felix.scr.annotations.Activate;
36import org.apache.felix.scr.annotations.Component;
37import org.apache.felix.scr.annotations.Deactivate;
38import org.apache.felix.scr.annotations.Reference;
39import org.apache.felix.scr.annotations.ReferenceCardinality;
40import org.apache.felix.scr.annotations.Service;
Bob zhou59a21062016-05-12 19:40:05 +080041import org.onlab.packet.ARP;
42import org.onlab.packet.Ethernet;
43import org.onlab.packet.IPv4;
jiangruic69a7fd2015-11-19 15:40:01 +080044import org.onlab.packet.Ip4Address;
lishuai6c56f5e2015-11-17 16:38:19 +080045import org.onlab.packet.IpAddress;
Bob zhou59a21062016-05-12 19:40:05 +080046import org.onlab.packet.IpPrefix;
lishuai6c56f5e2015-11-17 16:38:19 +080047import org.onlab.packet.MacAddress;
48import org.onlab.util.KryoNamespace;
49import org.onosproject.core.ApplicationId;
50import org.onosproject.core.CoreService;
51import org.onosproject.mastership.MastershipService;
lishuai858efd32015-12-04 14:30:36 +080052import org.onosproject.net.AnnotationKeys;
Bob zhou59a21062016-05-12 19:40:05 +080053import org.onosproject.net.ConnectPoint;
lishuai6c56f5e2015-11-17 16:38:19 +080054import org.onosproject.net.Device;
55import org.onosproject.net.DeviceId;
56import org.onosproject.net.Host;
lishuai858efd32015-12-04 14:30:36 +080057import org.onosproject.net.HostId;
lishuai6c56f5e2015-11-17 16:38:19 +080058import org.onosproject.net.Port;
59import org.onosproject.net.PortNumber;
60import org.onosproject.net.behaviour.BridgeConfig;
61import org.onosproject.net.behaviour.BridgeDescription;
jiangruic69a7fd2015-11-19 15:40:01 +080062import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
lishuai6c56f5e2015-11-17 16:38:19 +080063import org.onosproject.net.config.NetworkConfigService;
64import org.onosproject.net.config.basics.BasicDeviceConfig;
Phaneendra Mandab212bc92016-07-08 16:50:11 +053065import org.onosproject.net.config.basics.BasicHostConfig;
lishuai6c56f5e2015-11-17 16:38:19 +080066import org.onosproject.net.device.DeviceEvent;
67import org.onosproject.net.device.DeviceListener;
68import org.onosproject.net.device.DeviceService;
69import org.onosproject.net.driver.DriverHandler;
70import org.onosproject.net.driver.DriverService;
jiangruic69a7fd2015-11-19 15:40:01 +080071import org.onosproject.net.flow.DefaultTrafficTreatment;
Bob zhou59a21062016-05-12 19:40:05 +080072import org.onosproject.net.flow.FlowEntry;
73import org.onosproject.net.flow.FlowRuleService;
74import org.onosproject.net.flow.TrafficSelector;
75import org.onosproject.net.flow.TrafficTreatment;
jiangruic69a7fd2015-11-19 15:40:01 +080076import org.onosproject.net.flow.TrafficTreatment.Builder;
Bob zhou59a21062016-05-12 19:40:05 +080077import org.onosproject.net.flow.criteria.Criterion;
jiangruic69a7fd2015-11-19 15:40:01 +080078import org.onosproject.net.flow.instructions.ExtensionTreatment;
lishuai6c56f5e2015-11-17 16:38:19 +080079import org.onosproject.net.flowobjective.Objective;
jiangruic69a7fd2015-11-19 15:40:01 +080080import org.onosproject.net.group.DefaultGroupBucket;
81import org.onosproject.net.group.DefaultGroupDescription;
82import org.onosproject.net.group.DefaultGroupKey;
83import org.onosproject.net.group.GroupBucket;
84import org.onosproject.net.group.GroupBuckets;
85import org.onosproject.net.group.GroupDescription;
86import org.onosproject.net.group.GroupKey;
87import org.onosproject.net.group.GroupService;
lishuai6c56f5e2015-11-17 16:38:19 +080088import org.onosproject.net.host.HostEvent;
89import org.onosproject.net.host.HostListener;
90import org.onosproject.net.host.HostService;
Bob zhou59a21062016-05-12 19:40:05 +080091import org.onosproject.net.packet.DefaultOutboundPacket;
92import org.onosproject.net.packet.InboundPacket;
93import org.onosproject.net.packet.OutboundPacket;
94import org.onosproject.net.packet.PacketContext;
95import org.onosproject.net.packet.PacketProcessor;
96import org.onosproject.net.packet.PacketService;
lishuai6c56f5e2015-11-17 16:38:19 +080097import org.onosproject.store.serializers.KryoNamespaces;
lishuaib43dbf72016-01-06 11:11:35 +080098import org.onosproject.store.service.ConsistentMap;
lishuai6c56f5e2015-11-17 16:38:19 +080099import org.onosproject.store.service.EventuallyConsistentMap;
100import org.onosproject.store.service.LogicalClockService;
lishuaib43dbf72016-01-06 11:11:35 +0800101import org.onosproject.store.service.Serializer;
lishuai6c56f5e2015-11-17 16:38:19 +0800102import org.onosproject.store.service.StorageService;
lishuaib43dbf72016-01-06 11:11:35 +0800103import org.onosproject.store.service.Versioned;
Jonathan Hart51539b82015-10-29 09:53:04 -0700104import org.onosproject.vtn.manager.VtnService;
lishuai858efd32015-12-04 14:30:36 +0800105import org.onosproject.vtn.table.ArpService;
lishuai6c56f5e2015-11-17 16:38:19 +0800106import org.onosproject.vtn.table.ClassifierService;
lishuai858efd32015-12-04 14:30:36 +0800107import org.onosproject.vtn.table.DnatService;
lishuai6c56f5e2015-11-17 16:38:19 +0800108import org.onosproject.vtn.table.L2ForwardService;
lishuai858efd32015-12-04 14:30:36 +0800109import org.onosproject.vtn.table.L3ForwardService;
110import org.onosproject.vtn.table.SnatService;
lishuai74f2d532015-12-10 17:03:34 +0800111import org.onosproject.vtn.table.impl.ArpServiceImpl;
lishuai6c56f5e2015-11-17 16:38:19 +0800112import org.onosproject.vtn.table.impl.ClassifierServiceImpl;
lishuai74f2d532015-12-10 17:03:34 +0800113import org.onosproject.vtn.table.impl.DnatServiceImpl;
lishuai6c56f5e2015-11-17 16:38:19 +0800114import org.onosproject.vtn.table.impl.L2ForwardServiceImpl;
lishuai74f2d532015-12-10 17:03:34 +0800115import org.onosproject.vtn.table.impl.L3ForwardServiceImpl;
116import org.onosproject.vtn.table.impl.SnatServiceImpl;
lishuai6c56f5e2015-11-17 16:38:19 +0800117import org.onosproject.vtn.util.DataPathIdGenerator;
Bob zhou59a21062016-05-12 19:40:05 +0800118import org.onosproject.vtn.util.IpUtil;
lishuai6c56f5e2015-11-17 16:38:19 +0800119import org.onosproject.vtn.util.VtnConfig;
120import org.onosproject.vtn.util.VtnData;
lishuai7dc63d92015-11-27 17:15:25 +0800121import org.onosproject.vtnrsc.AllowedAddressPair;
122import org.onosproject.vtnrsc.BindingHostId;
Bob zhou59a21062016-05-12 19:40:05 +0800123import org.onosproject.vtnrsc.DefaultFloatingIp;
lishuai7dc63d92015-11-27 17:15:25 +0800124import org.onosproject.vtnrsc.DefaultVirtualPort;
125import org.onosproject.vtnrsc.FixedIp;
lishuai858efd32015-12-04 14:30:36 +0800126import org.onosproject.vtnrsc.FloatingIp;
Bob zhou59a21062016-05-12 19:40:05 +0800127import org.onosproject.vtnrsc.FloatingIpId;
lishuaibee5eca2016-05-10 19:17:37 +0800128import org.onosproject.vtnrsc.RouterId;
lishuai858efd32015-12-04 14:30:36 +0800129import org.onosproject.vtnrsc.RouterInterface;
lishuai7dc63d92015-11-27 17:15:25 +0800130import org.onosproject.vtnrsc.SecurityGroup;
lishuai6c56f5e2015-11-17 16:38:19 +0800131import org.onosproject.vtnrsc.SegmentationId;
Wu wenbinacc10ea2016-05-06 16:48:56 +0800132import org.onosproject.vtnrsc.Subnet;
lishuai6c56f5e2015-11-17 16:38:19 +0800133import org.onosproject.vtnrsc.SubnetId;
134import org.onosproject.vtnrsc.TenantId;
135import org.onosproject.vtnrsc.TenantNetwork;
136import org.onosproject.vtnrsc.TenantNetworkId;
lishuai8798bbe2016-05-05 16:02:03 +0800137import org.onosproject.vtnrsc.TenantRouter;
lishuai6c56f5e2015-11-17 16:38:19 +0800138import org.onosproject.vtnrsc.VirtualPort;
139import org.onosproject.vtnrsc.VirtualPortId;
lishuai858efd32015-12-04 14:30:36 +0800140import org.onosproject.vtnrsc.event.VtnRscEvent;
141import org.onosproject.vtnrsc.event.VtnRscEventFeedback;
142import org.onosproject.vtnrsc.event.VtnRscListener;
143import org.onosproject.vtnrsc.floatingip.FloatingIpService;
lishuai858efd32015-12-04 14:30:36 +0800144import org.onosproject.vtnrsc.routerinterface.RouterInterfaceService;
145import org.onosproject.vtnrsc.service.VtnRscService;
146import org.onosproject.vtnrsc.subnet.SubnetService;
lishuai6c56f5e2015-11-17 16:38:19 +0800147import org.onosproject.vtnrsc.tenantnetwork.TenantNetworkService;
148import org.onosproject.vtnrsc.virtualport.VirtualPortService;
149import org.slf4j.Logger;
150
jiangruic69a7fd2015-11-19 15:40:01 +0800151import com.google.common.collect.Lists;
lishuai6c56f5e2015-11-17 16:38:19 +0800152import com.google.common.collect.Sets;
153
154/**
155 * Provides implementation of VTNService.
156 */
157@Component(immediate = true)
158@Service
Jonathan Hart51539b82015-10-29 09:53:04 -0700159public class VtnManager implements VtnService {
lishuai6c56f5e2015-11-17 16:38:19 +0800160 private final Logger log = getLogger(getClass());
161 private static final String APP_ID = "org.onosproject.app.vtn";
162
163 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
164 protected NetworkConfigService configService;
165
166 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
167 protected DeviceService deviceService;
168
169 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
170 protected HostService hostService;
171
172 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
173 protected CoreService coreService;
174
175 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
176 protected StorageService storageService;
177
178 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
179 protected TenantNetworkService tenantNetworkService;
180
181 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
182 protected VirtualPortService virtualPortService;
183
184 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
185 protected DriverService driverService;
186
187 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
188 protected LogicalClockService clockService;
189
190 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
191 protected MastershipService mastershipService;
192
jiangruic69a7fd2015-11-19 15:40:01 +0800193 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
194 protected GroupService groupService;
195
lishuai858efd32015-12-04 14:30:36 +0800196 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
197 protected SubnetService subnetService;
198
199 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
200 protected VtnRscService vtnRscService;
201
202 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
203 protected FloatingIpService floatingIpService;
204
205 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
lishuai858efd32015-12-04 14:30:36 +0800206 protected RouterInterfaceService routerInterfaceService;
207
Bob zhou59a21062016-05-12 19:40:05 +0800208 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
209 protected FlowRuleService flowRuleService;
210
Phaneendra Mandab212bc92016-07-08 16:50:11 +0530211 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
212 protected NetworkConfigService networkConfigService;
213
lishuai6c56f5e2015-11-17 16:38:19 +0800214 private ApplicationId appId;
215 private ClassifierService classifierService;
216 private L2ForwardService l2ForwardService;
lishuai858efd32015-12-04 14:30:36 +0800217 private ArpService arpService;
218 private L3ForwardService l3ForwardService;
219 private SnatService snatService;
220 private DnatService dnatService;
lishuai6c56f5e2015-11-17 16:38:19 +0800221
222 private final HostListener hostListener = new InnerHostListener();
223 private final DeviceListener deviceListener = new InnerDeviceListener();
lishuai858efd32015-12-04 14:30:36 +0800224 private final VtnRscListener l3EventListener = new VtnL3EventListener();
lishuai6c56f5e2015-11-17 16:38:19 +0800225
lishuaib43dbf72016-01-06 11:11:35 +0800226 private static final String EX_PORT_KEY = "exPortKey";
lishuai6c56f5e2015-11-17 16:38:19 +0800227 private static final String IFACEID = "ifaceid";
228 private static final String CONTROLLER_IP_KEY = "ipaddress";
229 public static final String DRIVER_NAME = "onosfw";
lishuai7dc63d92015-11-27 17:15:25 +0800230 private static final String VIRTUALPORT = "vtn-virtual-port";
lishuai6c56f5e2015-11-17 16:38:19 +0800231 private static final String SWITCHES_OF_CONTROLLER = "switchesOfController";
232 private static final String SWITCH_OF_LOCAL_HOST_PORTS = "switchOfLocalHostPorts";
lishuai8798bbe2016-05-05 16:02:03 +0800233 private static final String ROUTERINF_FLAG_OF_TENANTROUTER = "routerInfFlagOfTenantRouter";
lishuai858efd32015-12-04 14:30:36 +0800234 private static final String HOSTS_OF_SUBNET = "hostsOfSubnet";
235 private static final String EX_PORT_OF_DEVICE = "exPortOfDevice";
lishuaib43dbf72016-01-06 11:11:35 +0800236 private static final String EX_PORT_MAP = "exPortMap";
jiangruic69a7fd2015-11-19 15:40:01 +0800237 private static final String DEFAULT_IP = "0.0.0.0";
Bob zhou59a21062016-05-12 19:40:05 +0800238 private static final String FLOATINGSTORE = "vtn-floatingIp";
Wu wenbinacc10ea2016-05-06 16:48:56 +0800239 private static final String USERDATA_IP = "169.254.169.254";
lishuai858efd32015-12-04 14:30:36 +0800240 private static final int SUBNET_NUM = 2;
Bob zhou59a21062016-05-12 19:40:05 +0800241 private static final int SNAT_TABLE = 40;
242 private static final int SNAT_DEFAULT_RULE_PRIORITY = 0;
243 private static final byte[] ZERO_MAC_ADDRESS = MacAddress.ZERO.toBytes();
lishuai6c56f5e2015-11-17 16:38:19 +0800244
lishuai7dc63d92015-11-27 17:15:25 +0800245 private EventuallyConsistentMap<VirtualPortId, VirtualPort> vPortStore;
lishuai6c56f5e2015-11-17 16:38:19 +0800246 private EventuallyConsistentMap<IpAddress, Boolean> switchesOfController;
jiangruic69a7fd2015-11-19 15:40:01 +0800247 private EventuallyConsistentMap<DeviceId, NetworkOfLocalHostPorts> switchOfLocalHostPorts;
lishuai858efd32015-12-04 14:30:36 +0800248 private EventuallyConsistentMap<SubnetId, Map<HostId, Host>> hostsOfSubnet;
lishuai8798bbe2016-05-05 16:02:03 +0800249 private EventuallyConsistentMap<TenantRouter, Boolean> routerInfFlagOfTenantRouter;
lishuai858efd32015-12-04 14:30:36 +0800250 private EventuallyConsistentMap<DeviceId, Port> exPortOfDevice;
Bob zhou59a21062016-05-12 19:40:05 +0800251 private EventuallyConsistentMap<IpAddress, FloatingIp> floatingIpStore;
lishuaib43dbf72016-01-06 11:11:35 +0800252 private static ConsistentMap<String, String> exPortMap;
lishuai6c56f5e2015-11-17 16:38:19 +0800253
Bob zhou59a21062016-05-12 19:40:05 +0800254 private VtnL3PacketProcessor l3PacketProcessor = new VtnL3PacketProcessor();
255 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
256 protected PacketService packetService;
257
lishuai6c56f5e2015-11-17 16:38:19 +0800258 @Activate
259 public void activate() {
260 appId = coreService.registerApplication(APP_ID);
261 classifierService = new ClassifierServiceImpl(appId);
262 l2ForwardService = new L2ForwardServiceImpl(appId);
lishuai74f2d532015-12-10 17:03:34 +0800263 arpService = new ArpServiceImpl(appId);
264 l3ForwardService = new L3ForwardServiceImpl(appId);
265 snatService = new SnatServiceImpl(appId);
266 dnatService = new DnatServiceImpl(appId);
lishuai6c56f5e2015-11-17 16:38:19 +0800267
268 deviceService.addListener(deviceListener);
269 hostService.addListener(hostListener);
lishuai74f2d532015-12-10 17:03:34 +0800270 vtnRscService.addListener(l3EventListener);
lishuai6c56f5e2015-11-17 16:38:19 +0800271
272 KryoNamespace.Builder serializer = KryoNamespace.newBuilder()
273 .register(KryoNamespaces.API)
274 .register(NetworkOfLocalHostPorts.class)
275 .register(TenantNetworkId.class)
276 .register(Host.class)
277 .register(TenantNetwork.class)
Bob zhou59a21062016-05-12 19:40:05 +0800278 .register(TenantNetworkId.class)
lishuai6c56f5e2015-11-17 16:38:19 +0800279 .register(TenantId.class)
lishuai7dc63d92015-11-27 17:15:25 +0800280 .register(SubnetId.class)
281 .register(VirtualPortId.class)
282 .register(VirtualPort.State.class)
283 .register(AllowedAddressPair.class)
284 .register(FixedIp.class)
Bob zhou59a21062016-05-12 19:40:05 +0800285 .register(FloatingIp.class)
286 .register(FloatingIpId.class)
287 .register(FloatingIp.Status.class)
288 .register(UUID.class)
289 .register(DefaultFloatingIp.class)
lishuai7dc63d92015-11-27 17:15:25 +0800290 .register(BindingHostId.class)
291 .register(SecurityGroup.class)
292 .register(IpAddress.class)
lishuaibee5eca2016-05-10 19:17:37 +0800293 .register(DefaultVirtualPort.class)
294 .register(RouterId.class)
295 .register(TenantRouter.class);
Bob zhou59a21062016-05-12 19:40:05 +0800296 floatingIpStore = storageService
297 .<IpAddress, FloatingIp>eventuallyConsistentMapBuilder()
298 .withName(FLOATINGSTORE).withSerializer(serializer)
299 .withTimestampProvider((k, v) -> clockService.getTimestamp())
300 .build();
lishuai7dc63d92015-11-27 17:15:25 +0800301
302 vPortStore = storageService
303 .<VirtualPortId, VirtualPort>eventuallyConsistentMapBuilder()
304 .withName(VIRTUALPORT).withSerializer(serializer)
305 .withTimestampProvider((k, v) -> clockService.getTimestamp())
306 .build();
lishuai6c56f5e2015-11-17 16:38:19 +0800307
308 switchesOfController = storageService
309 .<IpAddress, Boolean>eventuallyConsistentMapBuilder()
310 .withName(SWITCHES_OF_CONTROLLER).withSerializer(serializer)
311 .withTimestampProvider((k, v) -> clockService.getTimestamp())
312 .build();
313
314 switchOfLocalHostPorts = storageService
jiangruic69a7fd2015-11-19 15:40:01 +0800315 .<DeviceId, NetworkOfLocalHostPorts>eventuallyConsistentMapBuilder()
316 .withName(SWITCH_OF_LOCAL_HOST_PORTS).withSerializer(serializer)
317 .withTimestampProvider((k, v) -> clockService.getTimestamp())
lishuai6c56f5e2015-11-17 16:38:19 +0800318 .build();
319
lishuai858efd32015-12-04 14:30:36 +0800320 hostsOfSubnet = storageService
321 .<SubnetId, Map<HostId, Host>>eventuallyConsistentMapBuilder()
322 .withName(HOSTS_OF_SUBNET).withSerializer(serializer)
323 .withTimestampProvider((k, v) -> clockService.getTimestamp())
324 .build();
325
lishuai8798bbe2016-05-05 16:02:03 +0800326 routerInfFlagOfTenantRouter = storageService
327 .<TenantRouter, Boolean>eventuallyConsistentMapBuilder()
328 .withName(ROUTERINF_FLAG_OF_TENANTROUTER).withSerializer(serializer)
lishuai858efd32015-12-04 14:30:36 +0800329 .withTimestampProvider((k, v) -> clockService.getTimestamp())
330 .build();
331
332 exPortOfDevice = storageService
333 .<DeviceId, Port>eventuallyConsistentMapBuilder()
334 .withName(EX_PORT_OF_DEVICE).withSerializer(serializer)
335 .withTimestampProvider((k, v) -> clockService.getTimestamp())
336 .build();
337
lishuaib43dbf72016-01-06 11:11:35 +0800338 exPortMap = storageService
339 .<String, String>consistentMapBuilder()
340 .withName(EX_PORT_MAP)
341 .withApplicationId(appId)
342 .withPurgeOnUninstall()
343 .withSerializer(Serializer.using(Arrays.asList(KryoNamespaces.API)))
344 .build();
345
Bob zhou59a21062016-05-12 19:40:05 +0800346 packetService.addProcessor(l3PacketProcessor, PacketProcessor.director(0));
lishuai6c56f5e2015-11-17 16:38:19 +0800347 log.info("Started");
348 }
349
350 @Deactivate
351 public void deactivate() {
352 deviceService.removeListener(deviceListener);
353 hostService.removeListener(hostListener);
lishuai858efd32015-12-04 14:30:36 +0800354 vtnRscService.removeListener(l3EventListener);
lishuai6c56f5e2015-11-17 16:38:19 +0800355 log.info("Stopped");
356 }
357
358 @Override
359 public void onControllerDetected(Device controllerDevice) {
360 if (controllerDevice == null) {
361 log.error("The controller device is null");
362 return;
363 }
364 String localIpAddress = controllerDevice.annotations()
365 .value(CONTROLLER_IP_KEY);
366 IpAddress localIp = IpAddress.valueOf(localIpAddress);
367 DeviceId controllerDeviceId = controllerDevice.id();
368 DriverHandler handler = driverService.createHandler(controllerDeviceId);
369 if (mastershipService.isLocalMaster(controllerDeviceId)) {
370 // Get DataPathIdGenerator
371 String ipaddress = controllerDevice.annotations().value("ipaddress");
372 DataPathIdGenerator dpidGenerator = DataPathIdGenerator.builder()
373 .addIpAddress(ipaddress).build();
374 DeviceId deviceId = dpidGenerator.getDeviceId();
375 String dpid = dpidGenerator.getDpId();
376 // Inject pipeline driver name
377 BasicDeviceConfig config = configService.addConfig(deviceId,
378 BasicDeviceConfig.class);
379 config.driver(DRIVER_NAME);
380 configService.applyConfig(deviceId, BasicDeviceConfig.class, config.node());
381 // Add Bridge
lishuaib43dbf72016-01-06 11:11:35 +0800382 Versioned<String> exPortVersioned = exPortMap.get(EX_PORT_KEY);
383 if (exPortVersioned != null) {
384 VtnConfig.applyBridgeConfig(handler, dpid, exPortVersioned.value());
385 log.info("A new ovs is created in node {}", localIp.toString());
386 }
lishuai6c56f5e2015-11-17 16:38:19 +0800387 switchesOfController.put(localIp, true);
388 }
389 // Create tunnel in br-int on all controllers
390 programTunnelConfig(controllerDeviceId, localIp, handler);
391 }
392
393 @Override
394 public void onControllerVanished(Device controllerDevice) {
395 if (controllerDevice == null) {
396 log.error("The device is null");
397 return;
398 }
399 String dstIp = controllerDevice.annotations().value(CONTROLLER_IP_KEY);
400 IpAddress dstIpAddress = IpAddress.valueOf(dstIp);
401 DeviceId controllerDeviceId = controllerDevice.id();
402 if (mastershipService.isLocalMaster(controllerDeviceId)) {
403 switchesOfController.remove(dstIpAddress);
404 }
lishuai6c56f5e2015-11-17 16:38:19 +0800405 }
406
407 @Override
408 public void onOvsDetected(Device device) {
lishuaibee5eca2016-05-10 19:17:37 +0800409 if (device == null) {
410 log.error("The device is null");
411 return;
412 }
413 if (!mastershipService.isLocalMaster(device.id())) {
414 return;
415 }
lishuai6c56f5e2015-11-17 16:38:19 +0800416 // Create tunnel out flow rules
417 applyTunnelOut(device, Objective.Operation.ADD);
lishuaibee5eca2016-05-10 19:17:37 +0800418 // apply L3 arp flows
419 Iterable<RouterInterface> interfaces = routerInterfaceService
420 .getRouterInterfaces();
421 interfaces.forEach(routerInf -> {
422 VirtualPort gwPort = virtualPortService.getPort(routerInf.portId());
423 if (gwPort == null) {
424 gwPort = VtnData.getPort(vPortStore, routerInf.portId());
425 }
426 applyL3ArpFlows(device.id(), gwPort, Objective.Operation.ADD);
427 });
lishuai6c56f5e2015-11-17 16:38:19 +0800428 }
429
430 @Override
431 public void onOvsVanished(Device device) {
lishuaibee5eca2016-05-10 19:17:37 +0800432 if (device == null) {
433 log.error("The device is null");
434 return;
435 }
436 if (!mastershipService.isLocalMaster(device.id())) {
437 return;
438 }
lishuai6c56f5e2015-11-17 16:38:19 +0800439 // Remove Tunnel out flow rules
440 applyTunnelOut(device, Objective.Operation.REMOVE);
lishuaibee5eca2016-05-10 19:17:37 +0800441 // apply L3 arp flows
442 Iterable<RouterInterface> interfaces = routerInterfaceService
443 .getRouterInterfaces();
444 interfaces.forEach(routerInf -> {
445 VirtualPort gwPort = virtualPortService.getPort(routerInf.portId());
446 if (gwPort == null) {
447 gwPort = VtnData.getPort(vPortStore, routerInf.portId());
448 }
449 applyL3ArpFlows(device.id(), gwPort, Objective.Operation.REMOVE);
450 });
lishuai6c56f5e2015-11-17 16:38:19 +0800451 }
452
453 @Override
454 public void onHostDetected(Host host) {
lishuai858efd32015-12-04 14:30:36 +0800455 DeviceId deviceId = host.location().deviceId();
456 if (!mastershipService.isLocalMaster(deviceId)) {
457 return;
458 }
459 String ifaceId = host.annotations().value(IFACEID);
460 if (ifaceId == null) {
461 log.error("The ifaceId of Host is null");
462 return;
463 }
lishuaib43dbf72016-01-06 11:11:35 +0800464 programSffAndClassifierHost(host, Objective.Operation.ADD);
lishuai6c56f5e2015-11-17 16:38:19 +0800465 // apply L2 openflow rules
466 applyHostMonitoredL2Rules(host, Objective.Operation.ADD);
lishuai858efd32015-12-04 14:30:36 +0800467 // apply L3 openflow rules
468 applyHostMonitoredL3Rules(host, Objective.Operation.ADD);
lishuai6c56f5e2015-11-17 16:38:19 +0800469 }
470
471 @Override
472 public void onHostVanished(Host host) {
lishuai858efd32015-12-04 14:30:36 +0800473 DeviceId deviceId = host.location().deviceId();
474 if (!mastershipService.isLocalMaster(deviceId)) {
475 return;
476 }
477 String ifaceId = host.annotations().value(IFACEID);
478 if (ifaceId == null) {
479 log.error("The ifaceId of Host is null");
480 return;
481 }
lishuaib43dbf72016-01-06 11:11:35 +0800482 programSffAndClassifierHost(host, Objective.Operation.REMOVE);
lishuai6c56f5e2015-11-17 16:38:19 +0800483 // apply L2 openflow rules
484 applyHostMonitoredL2Rules(host, Objective.Operation.REMOVE);
lishuai858efd32015-12-04 14:30:36 +0800485 // apply L3 openflow rules
486 applyHostMonitoredL3Rules(host, Objective.Operation.REMOVE);
lishuai590d93a2015-12-11 13:05:14 +0800487 VirtualPortId virtualPortId = VirtualPortId.portId(ifaceId);
488 vPortStore.remove(virtualPortId);
lishuai6c56f5e2015-11-17 16:38:19 +0800489 }
490
491 private void programTunnelConfig(DeviceId localDeviceId, IpAddress localIp,
492 DriverHandler localHandler) {
jiangruic69a7fd2015-11-19 15:40:01 +0800493 if (mastershipService.isLocalMaster(localDeviceId)) {
Hyunsun Moondd14e8e2016-06-09 16:17:32 -0700494 VtnConfig.applyTunnelConfig(localHandler, localIp);
jiangruic69a7fd2015-11-19 15:40:01 +0800495 log.info("Add tunnel on {}", localIp);
496 }
lishuai6c56f5e2015-11-17 16:38:19 +0800497 }
498
499 private void applyTunnelOut(Device device, Objective.Operation type) {
lishuai6c56f5e2015-11-17 16:38:19 +0800500 String controllerIp = VtnData.getControllerIpOfSwitch(device);
501 if (controllerIp == null) {
502 log.error("Can't find controller of device: {}",
503 device.id().toString());
504 return;
505 }
506 IpAddress ipAddress = IpAddress.valueOf(controllerIp);
507 if (!switchesOfController.containsKey(ipAddress)) {
508 log.error("Can't find controller of device: {}",
509 device.id().toString());
510 return;
511 }
512 if (type == Objective.Operation.ADD) {
lishuaia3e32342015-12-07 10:59:17 +0800513 // Save external port
514 Port export = getExPort(device.id());
lishuaid6f0c9e2015-12-16 11:40:01 +0800515 if (export != null) {
Bob zhou59a21062016-05-12 19:40:05 +0800516 classifierService.programExportPortArpClassifierRules(export,
517 device.id(),
518 type);
lishuaid6f0c9e2015-12-16 11:40:01 +0800519 exPortOfDevice.put(device.id(), export);
520 }
lishuai6c56f5e2015-11-17 16:38:19 +0800521 switchOfLocalHostPorts.put(device.id(), new NetworkOfLocalHostPorts());
522 } else if (type == Objective.Operation.REMOVE) {
lishuaia3e32342015-12-07 10:59:17 +0800523 exPortOfDevice.remove(device.id());
lishuai6c56f5e2015-11-17 16:38:19 +0800524 switchOfLocalHostPorts.remove(device.id());
525 }
526 Iterable<Device> devices = deviceService.getAvailableDevices();
527 DeviceId localControllerId = VtnData.getControllerId(device, devices);
528 DriverHandler handler = driverService.createHandler(localControllerId);
529 Set<PortNumber> ports = VtnConfig.getPortNumbers(handler);
530 Iterable<Host> allHosts = hostService.getHosts();
jiangruic69a7fd2015-11-19 15:40:01 +0800531 String tunnelName = "vxlan-" + DEFAULT_IP;
lishuai6c56f5e2015-11-17 16:38:19 +0800532 if (allHosts != null) {
Sho SHIMIZUa09e1bb2016-08-01 14:25:25 -0700533 Sets.newHashSet(allHosts).forEach(host -> {
lishuai6c56f5e2015-11-17 16:38:19 +0800534 MacAddress hostMac = host.mac();
535 String ifaceId = host.annotations().value(IFACEID);
536 if (ifaceId == null) {
537 log.error("The ifaceId of Host is null");
538 return;
539 }
540 VirtualPortId virtualPortId = VirtualPortId.portId(ifaceId);
541 VirtualPort virtualPort = virtualPortService
542 .getPort(virtualPortId);
543 TenantNetwork network = tenantNetworkService
544 .getNetwork(virtualPort.networkId());
545 SegmentationId segmentationId = network.segmentationId();
546 DeviceId remoteDeviceId = host.location().deviceId();
547 Device remoteDevice = deviceService.getDevice(remoteDeviceId);
548 String remoteControllerIp = VtnData
549 .getControllerIpOfSwitch(remoteDevice);
550 if (remoteControllerIp == null) {
551 log.error("Can't find remote controller of device: {}",
Sho SHIMIZUa09e1bb2016-08-01 14:25:25 -0700552 remoteDeviceId.toString());
lishuai6c56f5e2015-11-17 16:38:19 +0800553 return;
554 }
555 IpAddress remoteIpAddress = IpAddress
556 .valueOf(remoteControllerIp);
lishuai6c56f5e2015-11-17 16:38:19 +0800557 ports.stream()
558 .filter(p -> p.name().equalsIgnoreCase(tunnelName))
559 .forEach(p -> {
Sho SHIMIZUa09e1bb2016-08-01 14:25:25 -0700560 l2ForwardService
561 .programTunnelOut(device.id(), segmentationId, p,
562 hostMac, type, remoteIpAddress);
563 });
lishuai6c56f5e2015-11-17 16:38:19 +0800564 });
565 }
566 }
567
lishuaib43dbf72016-01-06 11:11:35 +0800568 private void programSffAndClassifierHost(Host host, Objective.Operation type) {
569 DeviceId deviceId = host.location().deviceId();
570 String ifaceId = host.annotations().value(IFACEID);
571 VirtualPortId virtualPortId = VirtualPortId.portId(ifaceId);
572 VirtualPort virtualPort = virtualPortService.getPort(virtualPortId);
573 if (virtualPort == null) {
574 virtualPort = VtnData.getPort(vPortStore, virtualPortId);
575 }
576 TenantId tenantId = virtualPort.tenantId();
577 if (Objective.Operation.ADD == type) {
578 vtnRscService.addDeviceIdOfOvsMap(virtualPortId, tenantId, deviceId);
579 } else if (Objective.Operation.REMOVE == type) {
580 vtnRscService.removeDeviceIdOfOvsMap(host, tenantId, deviceId);
581 }
582 }
583
lishuai6c56f5e2015-11-17 16:38:19 +0800584 private void applyHostMonitoredL2Rules(Host host, Objective.Operation type) {
585 DeviceId deviceId = host.location().deviceId();
586 if (!mastershipService.isLocalMaster(deviceId)) {
587 return;
588 }
589 String ifaceId = host.annotations().value(IFACEID);
590 if (ifaceId == null) {
591 log.error("The ifaceId of Host is null");
592 return;
593 }
594 VirtualPortId virtualPortId = VirtualPortId.portId(ifaceId);
595 VirtualPort virtualPort = virtualPortService.getPort(virtualPortId);
596 if (virtualPort == null) {
lishuai858efd32015-12-04 14:30:36 +0800597 virtualPort = VtnData.getPort(vPortStore, virtualPortId);
lishuai6c56f5e2015-11-17 16:38:19 +0800598 }
lishuai74f2d532015-12-10 17:03:34 +0800599 Iterator<FixedIp> fixip = virtualPort.fixedIps().iterator();
600 SubnetId subnetId = null;
601 if (fixip.hasNext()) {
602 subnetId = fixip.next().subnetId();
603 }
604 if (subnetId != null) {
605 Map<HostId, Host> hosts = new ConcurrentHashMap();
606 if (hostsOfSubnet.get(subnetId) != null) {
607 hosts = hostsOfSubnet.get(subnetId);
608 }
609 if (type == Objective.Operation.ADD) {
610 hosts.put(host.id(), host);
611 hostsOfSubnet.put(subnetId, hosts);
612 } else if (type == Objective.Operation.REMOVE) {
613 hosts.remove(host.id());
614 if (hosts.size() != 0) {
615 hostsOfSubnet.put(subnetId, hosts);
616 } else {
617 hostsOfSubnet.remove(subnetId);
618 }
619 }
620 }
lishuai6c56f5e2015-11-17 16:38:19 +0800621
622 Iterable<Device> devices = deviceService.getAvailableDevices();
623 PortNumber inPort = host.location().port();
624 MacAddress mac = host.mac();
625 Device device = deviceService.getDevice(deviceId);
626 String controllerIp = VtnData.getControllerIpOfSwitch(device);
627 IpAddress ipAddress = IpAddress.valueOf(controllerIp);
628 TenantNetwork network = tenantNetworkService.getNetwork(virtualPort.networkId());
629 if (network == null) {
630 log.error("Can't find network of the host");
631 return;
632 }
633 SegmentationId segmentationId = network.segmentationId();
634 // Get all the tunnel PortNumber in the current node
635 Iterable<Port> ports = deviceService.getPorts(deviceId);
636 Collection<PortNumber> localTunnelPorts = VtnData.getLocalTunnelPorts(ports);
637 // Get all the local vm's PortNumber in the current node
638 Map<TenantNetworkId, Set<PortNumber>> localHostPorts = switchOfLocalHostPorts
jiangruic69a7fd2015-11-19 15:40:01 +0800639 .get(deviceId).getNetworkOfLocalHostPorts();
lishuai6c56f5e2015-11-17 16:38:19 +0800640 Set<PortNumber> networkOflocalHostPorts = localHostPorts.get(network.id());
jiangruic69a7fd2015-11-19 15:40:01 +0800641 for (PortNumber p : localTunnelPorts) {
642 programGroupTable(deviceId, appId, p, devices, type);
643 }
Wu wenbinacc10ea2016-05-06 16:48:56 +0800644 Subnet subnet = subnetService.getSubnet(subnetId);
645 String deviceOwner = virtualPort.deviceOwner();
646 if (deviceOwner != null) {
Jon Halla3fcf672017-03-28 16:53:22 -0700647 if ("network:dhcp".equalsIgnoreCase(deviceOwner)) {
Wu wenbinacc10ea2016-05-06 16:48:56 +0800648 Sets.newHashSet(devices).stream()
649 .filter(d -> d.type() == Device.Type.SWITCH)
650 .forEach(d -> {
651 if (subnet != null) {
652 IpAddress dstIp = IpAddress
653 .valueOf(USERDATA_IP);
654 classifierService
655 .programUserdataClassifierRules(d.id(),
656 subnet.cidr(),
657 dstIp,
658 mac,
659 segmentationId,
660 type);
661 }
662 });
663 }
664 }
lishuai6c56f5e2015-11-17 16:38:19 +0800665 if (type == Objective.Operation.ADD) {
lishuai7dc63d92015-11-27 17:15:25 +0800666 vPortStore.put(virtualPortId, virtualPort);
lishuai6c56f5e2015-11-17 16:38:19 +0800667 if (networkOflocalHostPorts == null) {
668 networkOflocalHostPorts = new HashSet<PortNumber>();
669 localHostPorts.putIfAbsent(network.id(), networkOflocalHostPorts);
670 }
671 networkOflocalHostPorts.add(inPort);
jiangruic69a7fd2015-11-19 15:40:01 +0800672 l2ForwardService.programLocalBcastRules(deviceId, segmentationId,
673 inPort, networkOflocalHostPorts,
674 localTunnelPorts,
675 type);
lishuai6c56f5e2015-11-17 16:38:19 +0800676 classifierService.programTunnelIn(deviceId, segmentationId,
677 localTunnelPorts,
678 type);
679 } else if (type == Objective.Operation.REMOVE) {
jiangruic69a7fd2015-11-19 15:40:01 +0800680 if (networkOflocalHostPorts != null) {
681 l2ForwardService.programLocalBcastRules(deviceId, segmentationId,
682 inPort, networkOflocalHostPorts,
683 localTunnelPorts,
684 type);
685 networkOflocalHostPorts.remove(inPort);
686 if (networkOflocalHostPorts.isEmpty()) {
687 classifierService.programTunnelIn(deviceId, segmentationId,
688 localTunnelPorts,
689 type);
690 switchOfLocalHostPorts.get(deviceId).getNetworkOfLocalHostPorts()
691 .remove(virtualPort.networkId());
692 }
lishuai6c56f5e2015-11-17 16:38:19 +0800693 }
694 }
695
jiangruic69a7fd2015-11-19 15:40:01 +0800696 l2ForwardService.programLocalOut(deviceId, segmentationId, inPort, mac,
697 type);
698
lishuai6c56f5e2015-11-17 16:38:19 +0800699 l2ForwardService.programTunnelBcastRules(deviceId, segmentationId,
700 networkOflocalHostPorts,
701 localTunnelPorts,
702 type);
703
704 programTunnelOuts(devices, ipAddress, segmentationId, mac,
705 type);
706
707 classifierService.programLocalIn(deviceId, segmentationId, inPort, mac,
708 appId, type);
709 }
710
711 private void programTunnelOuts(Iterable<Device> devices,
712 IpAddress ipAddress,
713 SegmentationId segmentationId,
714 MacAddress dstMac,
715 Objective.Operation type) {
jiangruic69a7fd2015-11-19 15:40:01 +0800716 String tunnelName = "vxlan-" + DEFAULT_IP;
lishuai6c56f5e2015-11-17 16:38:19 +0800717 Sets.newHashSet(devices).stream()
jiangruic69a7fd2015-11-19 15:40:01 +0800718 .filter(d -> d.type() == Device.Type.CONTROLLER)
719 .filter(d -> !("ovsdb:" + ipAddress).equals(d.id().toString()))
720 .forEach(d -> {
lishuai6c56f5e2015-11-17 16:38:19 +0800721 DriverHandler handler = driverService.createHandler(d.id());
Hyunsun Moon1251e192016-06-07 16:57:05 -0700722 BridgeConfig bridgeConfig = handler.behaviour(BridgeConfig.class);
723 Collection<BridgeDescription> bridgeDescriptions = bridgeConfig.getBridges();
724 for (BridgeDescription sw : bridgeDescriptions) {
725 if (sw.name().equals(VtnConfig.DEFAULT_BRIDGE_NAME) &&
726 sw.deviceId().isPresent()) {
727 List<Port> ports = deviceService.getPorts(sw.deviceId().get());
728 ports.stream().filter(p -> p.annotations().value(AnnotationKeys.PORT_NAME)
729 .equalsIgnoreCase(tunnelName))
730 .forEach(p -> l2ForwardService.programTunnelOut(
731 sw.deviceId().get(), segmentationId, p.number(),
732 dstMac, type, ipAddress));
lishuai70304d12015-12-14 17:15:26 +0800733 break;
734 }
lishuai6c56f5e2015-11-17 16:38:19 +0800735 }
736 });
737 }
738
739 private class InnerDeviceListener implements DeviceListener {
740
741 @Override
742 public void event(DeviceEvent event) {
743 Device device = event.subject();
744 if (Device.Type.CONTROLLER == device.type()) {
745 if (DeviceEvent.Type.DEVICE_ADDED == event.type()) {
746 onControllerDetected(device);
747 }
748 if (DeviceEvent.Type.DEVICE_AVAILABILITY_CHANGED == event.type()) {
749 if (deviceService.isAvailable(device.id())) {
750 onControllerDetected(device);
751 } else {
752 onControllerVanished(device);
753 }
754 }
755 } else if (Device.Type.SWITCH == device.type()) {
756 if (DeviceEvent.Type.DEVICE_ADDED == event.type()) {
757 onOvsDetected(device);
758 }
759 if (DeviceEvent.Type.DEVICE_AVAILABILITY_CHANGED == event.type()) {
760 if (deviceService.isAvailable(device.id())) {
761 onOvsDetected(device);
762 } else {
763 onOvsVanished(device);
764 }
765 }
766 } else {
767 log.info("Do nothing for this device type");
768 }
769 }
770 }
771
772 private class InnerHostListener implements HostListener {
773
774 @Override
775 public void event(HostEvent event) {
776 Host host = event.subject();
777 if (HostEvent.Type.HOST_ADDED == event.type()) {
778 onHostDetected(host);
779 } else if (HostEvent.Type.HOST_REMOVED == event.type()) {
780 onHostVanished(host);
781 } else if (HostEvent.Type.HOST_UPDATED == event.type()) {
782 onHostVanished(host);
783 onHostDetected(host);
784 }
785 }
786
787 }
788
789 // Local Host Ports of Network.
790 private class NetworkOfLocalHostPorts {
791 private final Map<TenantNetworkId, Set<PortNumber>> networkOfLocalHostPorts =
792 new HashMap<TenantNetworkId, Set<PortNumber>>();
793
794 public Map<TenantNetworkId, Set<PortNumber>> getNetworkOfLocalHostPorts() {
795 return networkOfLocalHostPorts;
796 }
797 }
798
jiangruic69a7fd2015-11-19 15:40:01 +0800799 private void programGroupTable(DeviceId deviceId, ApplicationId appid,
800 PortNumber portNumber, Iterable<Device> devices, Objective.Operation type) {
801 if (type.equals(Objective.Operation.REMOVE)) {
802 return;
803 }
804
805 List<GroupBucket> buckets = Lists.newArrayList();
806 Sets.newHashSet(devices)
807 .stream()
808 .filter(d -> d.type() == Device.Type.CONTROLLER)
809 .filter(d -> !deviceId.equals(d.id()))
810 .forEach(d -> {
811 String ipAddress = d.annotations()
812 .value(CONTROLLER_IP_KEY);
813 Ip4Address dst = Ip4Address.valueOf(ipAddress);
814 Builder builder = DefaultTrafficTreatment.builder();
815
816 DriverHandler handler = driverService.createHandler(deviceId);
817 ExtensionTreatmentResolver resolver = handler.behaviour(ExtensionTreatmentResolver.class);
818 ExtensionTreatment treatment = resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type());
819 try {
820 treatment.setPropertyValue("tunnelDst", dst);
821 } catch (Exception e) {
822 log.error("Failed to get extension instruction to set tunnel dst {}", deviceId);
823 }
824
825 builder.extension(treatment, deviceId);
826 builder.setOutput(portNumber);
827 GroupBucket bucket = DefaultGroupBucket
828 .createAllGroupBucket(builder.build());
829 buckets.add(bucket);
830 });
831 final GroupKey key = new DefaultGroupKey(APP_ID.getBytes());
832 GroupDescription groupDescription = new DefaultGroupDescription(deviceId,
833 GroupDescription.Type.ALL,
834 new GroupBuckets(buckets),
835 key,
836 L2ForwardServiceImpl.GROUP_ID,
837 appid);
838 groupService.addGroup(groupDescription);
839 }
lishuai858efd32015-12-04 14:30:36 +0800840
841 private class VtnL3EventListener implements VtnRscListener {
842 @Override
843 public void event(VtnRscEvent event) {
844 VtnRscEventFeedback l3Feedback = event.subject();
845 if (VtnRscEvent.Type.ROUTER_INTERFACE_PUT == event.type()) {
846 onRouterInterfaceDetected(l3Feedback);
Phaneendra Mandab212bc92016-07-08 16:50:11 +0530847 } else if (VtnRscEvent.Type.ROUTER_INTERFACE_DELETE == event.type()) {
lishuai858efd32015-12-04 14:30:36 +0800848 onRouterInterfaceVanished(l3Feedback);
lishuai762df812016-01-08 11:51:15 +0800849 } else if (VtnRscEvent.Type.FLOATINGIP_BIND == event.type()) {
lishuai858efd32015-12-04 14:30:36 +0800850 onFloatingIpDetected(l3Feedback);
lishuai762df812016-01-08 11:51:15 +0800851 } else if (VtnRscEvent.Type.FLOATINGIP_UNBIND == event.type()) {
lishuai858efd32015-12-04 14:30:36 +0800852 onFloatingIpVanished(l3Feedback);
Phaneendra Mandab212bc92016-07-08 16:50:11 +0530853 } else if (VtnRscEvent.Type.VIRTUAL_PORT_PUT == event.type()) {
854 onVirtualPortCreated(l3Feedback);
855 } else if (VtnRscEvent.Type.VIRTUAL_PORT_DELETE == event.type()) {
856 onVirtualPortDeleted(l3Feedback);
lishuai858efd32015-12-04 14:30:36 +0800857 }
858 }
859
860 }
861
862 @Override
863 public void onRouterInterfaceDetected(VtnRscEventFeedback l3Feedback) {
864 Objective.Operation operation = Objective.Operation.ADD;
865 RouterInterface routerInf = l3Feedback.routerInterface();
lishuai762df812016-01-08 11:51:15 +0800866 VirtualPort gwPort = virtualPortService.getPort(routerInf.portId());
867 vPortStore.put(gwPort.portId(), gwPort);
lishuai858efd32015-12-04 14:30:36 +0800868 Iterable<RouterInterface> interfaces = routerInterfaceService
869 .getRouterInterfaces();
lishuai8798bbe2016-05-05 16:02:03 +0800870 Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces).stream()
871 .filter(r -> r.tenantId().equals(routerInf.tenantId()))
872 .filter(r -> r.routerId().equals(routerInf.routerId()))
lishuai858efd32015-12-04 14:30:36 +0800873 .collect(Collectors.toSet());
lishuai8798bbe2016-05-05 16:02:03 +0800874 TenantRouter tenantRouter = TenantRouter
875 .tenantRouter(routerInf.tenantId(), routerInf.routerId());
876 if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
lishuai858efd32015-12-04 14:30:36 +0800877 programRouterInterface(routerInf, operation);
878 } else {
879 if (interfacesSet.size() >= SUBNET_NUM) {
880 programInterfacesSet(interfacesSet, operation);
881 }
882 }
lishuaibee5eca2016-05-10 19:17:37 +0800883 // apply L3 arp flows
884 applyL3ArpFlows(null, gwPort, operation);
lishuai858efd32015-12-04 14:30:36 +0800885 }
886
887 @Override
888 public void onRouterInterfaceVanished(VtnRscEventFeedback l3Feedback) {
889 Objective.Operation operation = Objective.Operation.REMOVE;
890 RouterInterface routerInf = l3Feedback.routerInterface();
891 Iterable<RouterInterface> interfaces = routerInterfaceService
892 .getRouterInterfaces();
893 Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces)
894 .stream().filter(r -> r.tenantId().equals(routerInf.tenantId()))
895 .collect(Collectors.toSet());
lishuai8798bbe2016-05-05 16:02:03 +0800896 TenantRouter tenantRouter = TenantRouter
897 .tenantRouter(routerInf.tenantId(), routerInf.routerId());
898 if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
lishuai858efd32015-12-04 14:30:36 +0800899 programRouterInterface(routerInf, operation);
900 if (interfacesSet.size() == 1) {
lishuai8798bbe2016-05-05 16:02:03 +0800901 routerInfFlagOfTenantRouter.remove(tenantRouter);
Sho SHIMIZUa09e1bb2016-08-01 14:25:25 -0700902 interfacesSet.forEach(r -> {
lishuai858efd32015-12-04 14:30:36 +0800903 programRouterInterface(r, operation);
904 });
905 }
906 }
lishuai762df812016-01-08 11:51:15 +0800907 VirtualPort gwPort = virtualPortService.getPort(routerInf.portId());
908 if (gwPort == null) {
909 gwPort = VtnData.getPort(vPortStore, routerInf.portId());
910 }
911 vPortStore.remove(gwPort.portId());
lishuaibee5eca2016-05-10 19:17:37 +0800912 // apply L3 arp flows
913 applyL3ArpFlows(null, gwPort, operation);
lishuai858efd32015-12-04 14:30:36 +0800914 }
915
916 @Override
917 public void onFloatingIpDetected(VtnRscEventFeedback l3Feedback) {
Bob zhou59a21062016-05-12 19:40:05 +0800918 floatingIpStore.put(l3Feedback.floatingIp().floatingIp(),
919 l3Feedback.floatingIp());
lishuai762df812016-01-08 11:51:15 +0800920 programFloatingIpEvent(l3Feedback, VtnRscEvent.Type.FLOATINGIP_BIND);
lishuai858efd32015-12-04 14:30:36 +0800921 }
922
923 @Override
924 public void onFloatingIpVanished(VtnRscEventFeedback l3Feedback) {
Bob zhou59a21062016-05-12 19:40:05 +0800925 floatingIpStore.remove(l3Feedback.floatingIp().floatingIp());
lishuai762df812016-01-08 11:51:15 +0800926 programFloatingIpEvent(l3Feedback, VtnRscEvent.Type.FLOATINGIP_UNBIND);
lishuai858efd32015-12-04 14:30:36 +0800927 }
928
Phaneendra Mandab212bc92016-07-08 16:50:11 +0530929 public void onVirtualPortCreated(VtnRscEventFeedback l3Feedback) {
930 VirtualPort vPort = l3Feedback.virtualPort();
931 BasicHostConfig basicHostConfig = networkConfigService.addConfig(HostId.hostId(vPort.macAddress()),
932 BasicHostConfig.class);
933 Set<IpAddress> ips = new HashSet<>();
934 for (FixedIp fixedIp : vPort.fixedIps()) {
935 ips.add(fixedIp.ip());
936 }
937 basicHostConfig.setIps(ips).apply();
938 }
939
940 public void onVirtualPortDeleted(VtnRscEventFeedback l3Feedback) {
941 VirtualPort vPort = l3Feedback.virtualPort();
942 HostId hostId = HostId.hostId(vPort.macAddress());
943 BasicHostConfig basicHostConfig = networkConfigService.addConfig(hostId,
944 BasicHostConfig.class);
Phaneendra Mandac11e57a2017-05-03 19:03:21 +0530945 Set<IpAddress> oldIps = hostService.getHost(hostId).ipAddresses();
946 // Copy to a new set as oldIps is unmodifiable set.
947 Set<IpAddress> newIps = new HashSet<>();
948 newIps.addAll(oldIps);
Phaneendra Mandab212bc92016-07-08 16:50:11 +0530949 for (FixedIp fixedIp : vPort.fixedIps()) {
Phaneendra Mandac11e57a2017-05-03 19:03:21 +0530950 newIps.remove(fixedIp.ip());
Phaneendra Mandab212bc92016-07-08 16:50:11 +0530951 }
Phaneendra Mandac11e57a2017-05-03 19:03:21 +0530952 basicHostConfig.setIps(newIps).apply();
Phaneendra Mandab212bc92016-07-08 16:50:11 +0530953 }
954
lishuai858efd32015-12-04 14:30:36 +0800955 private void programInterfacesSet(Set<RouterInterface> interfacesSet,
956 Objective.Operation operation) {
957 int subnetVmNum = 0;
958 for (RouterInterface r : interfacesSet) {
959 // Get all the host of the subnet
960 Map<HostId, Host> hosts = hostsOfSubnet.get(r.subnetId());
lishuaid6f0c9e2015-12-16 11:40:01 +0800961 if (hosts != null && hosts.size() > 0) {
lishuai858efd32015-12-04 14:30:36 +0800962 subnetVmNum++;
963 if (subnetVmNum >= SUBNET_NUM) {
lishuai8798bbe2016-05-05 16:02:03 +0800964 TenantRouter tenantRouter = TenantRouter
965 .tenantRouter(r.tenantId(), r.routerId());
966 routerInfFlagOfTenantRouter.put(tenantRouter, true);
Sho SHIMIZUa09e1bb2016-08-01 14:25:25 -0700967 interfacesSet.forEach(f -> {
lishuai858efd32015-12-04 14:30:36 +0800968 programRouterInterface(f, operation);
969 });
970 break;
971 }
972 }
973 }
974 }
975
976 private void programRouterInterface(RouterInterface routerInf,
977 Objective.Operation operation) {
lishuai8798bbe2016-05-05 16:02:03 +0800978 TenantRouter tenantRouter = TenantRouter
979 .tenantRouter(routerInf.tenantId(), routerInf.routerId());
980 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
lishuai858efd32015-12-04 14:30:36 +0800981 // Get all the host of the subnet
982 Map<HostId, Host> hosts = hostsOfSubnet.get(routerInf.subnetId());
Sho SHIMIZUa09e1bb2016-08-01 14:25:25 -0700983 hosts.values().forEach(h -> {
lishuai858efd32015-12-04 14:30:36 +0800984 applyEastWestL3Flows(h, l3vni, operation);
985 });
986 }
987
lishuaibee5eca2016-05-10 19:17:37 +0800988 private void applyL3ArpFlows(DeviceId deviceId, VirtualPort gwPort,
989 Objective.Operation operation) {
990 IpAddress ip = null;
991 Iterator<FixedIp> gwIps = gwPort.fixedIps().iterator();
992 if (gwIps.hasNext()) {
993 ip = gwIps.next().ip();
994 }
995 IpAddress gwIp = ip;
996 MacAddress gwMac = gwPort.macAddress();
997 TenantNetwork network = tenantNetworkService
998 .getNetwork(gwPort.networkId());
999 if (deviceId != null) {
1000 // Arp rules
1001 DriverHandler handler = driverService.createHandler(deviceId);
1002 arpService.programArpRules(handler, deviceId, gwIp,
1003 network.segmentationId(), gwMac,
1004 operation);
1005 } else {
1006 Iterable<Device> devices = deviceService.getAvailableDevices();
1007 Sets.newHashSet(devices).stream()
1008 .filter(d -> Device.Type.SWITCH == d.type()).forEach(d -> {
1009 // Arp rules
1010 DriverHandler handler = driverService.createHandler(d.id());
1011 arpService.programArpRules(handler, d.id(), gwIp,
1012 network.segmentationId(), gwMac,
1013 operation);
1014 });
1015 }
1016 }
1017
lishuai858efd32015-12-04 14:30:36 +08001018 private void applyEastWestL3Flows(Host h, SegmentationId l3vni,
1019 Objective.Operation operation) {
1020 if (!mastershipService.isLocalMaster(h.location().deviceId())) {
1021 log.debug("not master device:{}", h.location().deviceId());
1022 return;
1023 }
1024 String ifaceId = h.annotations().value(IFACEID);
1025 VirtualPort hPort = virtualPortService
1026 .getPort(VirtualPortId.portId(ifaceId));
1027 if (hPort == null) {
1028 hPort = VtnData.getPort(vPortStore, VirtualPortId.portId(ifaceId));
1029 }
1030 IpAddress srcIp = null;
1031 IpAddress srcGwIp = null;
1032 MacAddress srcVmGwMac = null;
1033 SubnetId srcSubnetId = null;
1034 Iterator<FixedIp> srcIps = hPort.fixedIps().iterator();
1035 if (srcIps.hasNext()) {
1036 FixedIp fixedIp = srcIps.next();
1037 srcIp = fixedIp.ip();
1038 srcSubnetId = fixedIp.subnetId();
1039 srcGwIp = subnetService.getSubnet(srcSubnetId).gatewayIp();
1040 FixedIp fixedGwIp = FixedIp.fixedIp(srcSubnetId, srcGwIp);
1041 VirtualPort gwPort = virtualPortService.getPort(fixedGwIp);
1042 if (gwPort == null) {
1043 gwPort = VtnData.getPort(vPortStore, fixedGwIp);
1044 }
1045 srcVmGwMac = gwPort.macAddress();
1046 }
1047 TenantNetwork network = tenantNetworkService
1048 .getNetwork(hPort.networkId());
lishuaibee5eca2016-05-10 19:17:37 +08001049 IpAddress dstVmIP = srcIp;
1050 MacAddress dstVmGwMac = srcVmGwMac;
1051 TenantId tenantId = hPort.tenantId();
lishuai858efd32015-12-04 14:30:36 +08001052 // Classifier rules
lishuaibee5eca2016-05-10 19:17:37 +08001053 if (operation == Objective.Operation.ADD) {
1054 sendEastWestL3Flows(h, srcVmGwMac, l3vni, srcGwIp, network,
1055 dstVmIP, dstVmGwMac, operation);
1056 } else if (operation == Objective.Operation.REMOVE) {
1057 FloatingIp floatingIp = null;
1058 Iterable<FloatingIp> floatingIps = floatingIpService.getFloatingIps();
1059 Set<FloatingIp> floatingIpSet = Sets.newHashSet(floatingIps).stream()
1060 .filter(f -> f.tenantId().equals(tenantId))
1061 .collect(Collectors.toSet());
1062 for (FloatingIp f : floatingIpSet) {
1063 IpAddress fixedIp = f.fixedIp();
1064 if (fixedIp != null && fixedIp.equals(srcIp)) {
1065 floatingIp = f;
1066 break;
1067 }
1068 }
1069 if (floatingIp == null) {
1070 sendEastWestL3Flows(h, srcVmGwMac, l3vni, srcGwIp, network,
1071 dstVmIP, dstVmGwMac, operation);
1072 }
1073 }
1074 }
1075
1076 private void sendEastWestL3Flows(Host h, MacAddress srcVmGwMac,
1077 SegmentationId l3vni, IpAddress srcGwIp,
1078 TenantNetwork network, IpAddress dstVmIP,
1079 MacAddress dstVmGwMac,
1080 Objective.Operation operation) {
lishuai858efd32015-12-04 14:30:36 +08001081 classifierService
1082 .programL3InPortClassifierRules(h.location().deviceId(),
1083 h.location().port(), h.mac(),
1084 srcVmGwMac, l3vni, operation);
lishuaibee5eca2016-05-10 19:17:37 +08001085 classifierService
1086 .programArpClassifierRules(h.location().deviceId(),
1087 h.location().port(), srcGwIp,
1088 network.segmentationId(), operation);
lishuai858efd32015-12-04 14:30:36 +08001089 Iterable<Device> devices = deviceService.getAvailableDevices();
lishuai858efd32015-12-04 14:30:36 +08001090 Sets.newHashSet(devices).stream()
1091 .filter(d -> Device.Type.SWITCH == d.type()).forEach(d -> {
1092 // L3FWD rules
lishuaibee5eca2016-05-10 19:17:37 +08001093 l3ForwardService.programRouteRules(d.id(), l3vni, dstVmIP,
lishuai858efd32015-12-04 14:30:36 +08001094 network.segmentationId(),
lishuaibee5eca2016-05-10 19:17:37 +08001095 dstVmGwMac, h.mac(),
lishuai858efd32015-12-04 14:30:36 +08001096 operation);
1097 });
1098 }
1099
1100 private void programFloatingIpEvent(VtnRscEventFeedback l3Feedback,
1101 VtnRscEvent.Type type) {
1102 FloatingIp floaingIp = l3Feedback.floatingIp();
1103 if (floaingIp != null) {
1104 VirtualPortId vmPortId = floaingIp.portId();
1105 VirtualPort vmPort = virtualPortService.getPort(vmPortId);
1106 VirtualPort fipPort = virtualPortService
1107 .getPort(floaingIp.networkId(), floaingIp.floatingIp());
1108 if (vmPort == null) {
1109 vmPort = VtnData.getPort(vPortStore, vmPortId);
1110 }
1111 if (fipPort == null) {
1112 fipPort = VtnData.getPort(vPortStore, floaingIp.networkId(),
1113 floaingIp.floatingIp());
1114 }
1115 Set<Host> hostSet = hostService.getHostsByMac(vmPort.macAddress());
1116 Host host = null;
1117 for (Host h : hostSet) {
1118 String ifaceid = h.annotations().value(IFACEID);
1119 if (ifaceid != null && ifaceid.equals(vmPortId.portId())) {
1120 host = h;
1121 break;
1122 }
1123 }
1124 if (host != null && vmPort != null && fipPort != null) {
1125 DeviceId deviceId = host.location().deviceId();
1126 Port exPort = exPortOfDevice.get(deviceId);
lishuai8798bbe2016-05-05 16:02:03 +08001127 TenantRouter tenantRouter = TenantRouter
1128 .tenantRouter(floaingIp.tenantId(), floaingIp.routerId());
1129 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
lishuai858efd32015-12-04 14:30:36 +08001130 // Floating ip BIND
lishuai762df812016-01-08 11:51:15 +08001131 if (type == VtnRscEvent.Type.FLOATINGIP_BIND) {
lishuai16c05af2016-01-21 18:39:05 +08001132 vPortStore.put(fipPort.portId(), fipPort);
lishuaibee5eca2016-05-10 19:17:37 +08001133 applyNorthSouthL3Flows(deviceId, false, tenantRouter, host,
1134 vmPort, fipPort, floaingIp, l3vni,
1135 exPort, Objective.Operation.ADD);
lishuai762df812016-01-08 11:51:15 +08001136 } else if (type == VtnRscEvent.Type.FLOATINGIP_UNBIND) {
lishuai858efd32015-12-04 14:30:36 +08001137 // Floating ip UNBIND
lishuaibee5eca2016-05-10 19:17:37 +08001138 applyNorthSouthL3Flows(deviceId, false, tenantRouter, host,
1139 vmPort, fipPort, floaingIp, l3vni,
1140 exPort,
lishuai858efd32015-12-04 14:30:36 +08001141 Objective.Operation.REMOVE);
lishuai16c05af2016-01-21 18:39:05 +08001142 vPortStore.remove(fipPort.portId());
lishuai858efd32015-12-04 14:30:36 +08001143 }
1144 }
1145 }
1146 }
1147
lishuaibee5eca2016-05-10 19:17:37 +08001148 private void sendNorthSouthL3Flows(DeviceId deviceId, FloatingIp floatingIp,
1149 IpAddress dstVmGwIp,
1150 MacAddress dstVmGwMac,
1151 SegmentationId l3vni,
1152 TenantNetwork vmNetwork,
1153 VirtualPort vmPort, Host host,
1154 Objective.Operation operation) {
1155 l3ForwardService
1156 .programRouteRules(deviceId, l3vni, floatingIp.fixedIp(),
1157 vmNetwork.segmentationId(), dstVmGwMac,
1158 vmPort.macAddress(), operation);
1159 classifierService.programL3InPortClassifierRules(deviceId,
1160 host.location().port(),
1161 host.mac(), dstVmGwMac,
1162 l3vni, operation);
1163 classifierService.programArpClassifierRules(deviceId, host.location()
1164 .port(), dstVmGwIp, vmNetwork.segmentationId(), operation);
1165 }
1166
1167 private void applyNorthSouthL3Flows(DeviceId deviceId, boolean hostFlag,
1168 TenantRouter tenantRouter, Host host,
lishuai858efd32015-12-04 14:30:36 +08001169 VirtualPort vmPort, VirtualPort fipPort,
1170 FloatingIp floatingIp,
lishuai8798bbe2016-05-05 16:02:03 +08001171 SegmentationId l3vni, Port exPort,
lishuai858efd32015-12-04 14:30:36 +08001172 Objective.Operation operation) {
1173 if (!mastershipService.isLocalMaster(deviceId)) {
1174 log.debug("not master device:{}", deviceId);
1175 return;
1176 }
1177 List gwIpMac = getGwIpAndMac(vmPort);
1178 IpAddress dstVmGwIp = (IpAddress) gwIpMac.get(0);
1179 MacAddress dstVmGwMac = (MacAddress) gwIpMac.get(1);
lishuai858efd32015-12-04 14:30:36 +08001180 TenantNetwork vmNetwork = tenantNetworkService
1181 .getNetwork(vmPort.networkId());
1182 TenantNetwork fipNetwork = tenantNetworkService
1183 .getNetwork(fipPort.networkId());
1184 // L3 downlink traffic flow
lishuaia3e32342015-12-07 10:59:17 +08001185 MacAddress exPortMac = MacAddress.valueOf(exPort.annotations()
1186 .value(AnnotationKeys.PORT_MAC));
lishuai858efd32015-12-04 14:30:36 +08001187 classifierService.programL3ExPortClassifierRules(deviceId, exPort.number(),
1188 floatingIp.floatingIp(), operation);
lishuai858efd32015-12-04 14:30:36 +08001189 dnatService.programRules(deviceId, floatingIp.floatingIp(),
Bob zhou59a21062016-05-12 19:40:05 +08001190 exPortMac, floatingIp.fixedIp(),
lishuai8798bbe2016-05-05 16:02:03 +08001191 l3vni, operation);
lishuai858efd32015-12-04 14:30:36 +08001192
Bob zhou59a21062016-05-12 19:40:05 +08001193 Subnet subnet = getSubnetOfFloatingIP(floatingIp);
1194 IpPrefix ipPrefix = subnet.cidr();
1195 snatService.programSnatSameSegmentUploadControllerRules(deviceId, l3vni,
1196 floatingIp.fixedIp(),
1197 floatingIp.floatingIp(),
1198 ipPrefix,
1199 operation);
lishuai858efd32015-12-04 14:30:36 +08001200 // L3 uplink traffic flow
lishuaibee5eca2016-05-10 19:17:37 +08001201 if (operation == Objective.Operation.ADD) {
1202 sendNorthSouthL3Flows(deviceId, floatingIp, dstVmGwIp, dstVmGwMac,
1203 l3vni, vmNetwork, vmPort, host, operation);
Bob zhou59a21062016-05-12 19:40:05 +08001204 l2ForwardService
1205 .programExternalOut(deviceId, fipNetwork.segmentationId(),
1206 exPort.number(), exPortMac, operation);
lishuaibee5eca2016-05-10 19:17:37 +08001207 } else if (operation == Objective.Operation.REMOVE) {
1208 if (hostFlag || (!hostFlag
1209 && routerInfFlagOfTenantRouter.get(tenantRouter) == null)) {
1210 sendNorthSouthL3Flows(deviceId, floatingIp, dstVmGwIp, dstVmGwMac,
1211 l3vni, vmNetwork, vmPort, host, operation);
1212 }
1213 Iterable<FloatingIp> floatingIps = floatingIpService.getFloatingIps();
1214 boolean exPortFlag = true;
1215 if (floatingIps != null) {
1216 Set<FloatingIp> floatingIpSet = Sets.newHashSet(floatingIps);
1217 for (FloatingIp fip : floatingIpSet) {
1218 if (fip.fixedIp() != null) {
1219 exPortFlag = false;
1220 break;
1221 }
1222 }
1223 }
1224 if (exPortFlag) {
Bob zhou59a21062016-05-12 19:40:05 +08001225 l2ForwardService.programExternalOut(deviceId,
1226 fipNetwork.segmentationId(),
1227 exPort.number(), exPortMac,
1228 operation);
lishuaibee5eca2016-05-10 19:17:37 +08001229 }
Bob zhou59a21062016-05-12 19:40:05 +08001230 removeRulesInSnat(deviceId, floatingIp.fixedIp());
lishuaibee5eca2016-05-10 19:17:37 +08001231 }
lishuai858efd32015-12-04 14:30:36 +08001232 }
1233
1234 private Port getExPort(DeviceId deviceId) {
1235 List<Port> ports = deviceService.getPorts(deviceId);
1236 Port exPort = null;
1237 for (Port port : ports) {
1238 String portName = port.annotations().value(AnnotationKeys.PORT_NAME);
lishuaib43dbf72016-01-06 11:11:35 +08001239 Versioned<String> exPortVersioned = exPortMap.get(EX_PORT_KEY);
1240 if (portName != null && exPortVersioned != null && portName.
1241 equals(exPortVersioned.value())) {
lishuai858efd32015-12-04 14:30:36 +08001242 exPort = port;
1243 break;
1244 }
1245 }
1246 return exPort;
1247 }
1248
1249 private List getGwIpAndMac(VirtualPort port) {
1250 List list = new ArrayList();
1251 MacAddress gwMac = null;
1252 SubnetId subnetId = null;
1253 IpAddress gwIp = null;
1254 Iterator<FixedIp> fixips = port.fixedIps().iterator();
1255 if (fixips.hasNext()) {
1256 FixedIp fixip = fixips.next();
1257 subnetId = fixip.subnetId();
1258 gwIp = subnetService.getSubnet(subnetId).gatewayIp();
1259 FixedIp fixedGwIp = FixedIp.fixedIp(fixip.subnetId(), gwIp);
1260 VirtualPort gwPort = virtualPortService.getPort(fixedGwIp);
1261 if (gwPort == null) {
1262 gwPort = VtnData.getPort(vPortStore, fixedGwIp);
1263 }
1264 gwMac = gwPort.macAddress();
1265 }
1266 list.add(gwIp);
1267 list.add(gwMac);
1268 return list;
1269 }
1270
lishuai858efd32015-12-04 14:30:36 +08001271 private void applyHostMonitoredL3Rules(Host host,
1272 Objective.Operation operation) {
1273 String ifaceId = host.annotations().value(IFACEID);
1274 DeviceId deviceId = host.location().deviceId();
1275 VirtualPortId portId = VirtualPortId.portId(ifaceId);
1276 VirtualPort port = virtualPortService.getPort(portId);
1277 if (port == null) {
1278 port = VtnData.getPort(vPortStore, portId);
1279 }
1280 TenantId tenantId = port.tenantId();
1281 Port exPort = exPortOfDevice.get(deviceId);
lishuai858efd32015-12-04 14:30:36 +08001282 Iterator<FixedIp> fixips = port.fixedIps().iterator();
1283 SubnetId sid = null;
1284 IpAddress hostIp = null;
1285 if (fixips.hasNext()) {
1286 FixedIp fixip = fixips.next();
1287 sid = fixip.subnetId();
1288 hostIp = fixip.ip();
1289 }
1290 final SubnetId subnetId = sid;
1291 // L3 internal network access to each other
1292 Iterable<RouterInterface> interfaces = routerInterfaceService
1293 .getRouterInterfaces();
lishuai8798bbe2016-05-05 16:02:03 +08001294 Set<RouterInterface> hostInterfaces = Sets.newHashSet(interfaces)
lishuai858efd32015-12-04 14:30:36 +08001295 .stream().filter(r -> r.tenantId().equals(tenantId))
lishuai8798bbe2016-05-05 16:02:03 +08001296 .filter(r -> r.subnetId().equals(subnetId))
lishuai858efd32015-12-04 14:30:36 +08001297 .collect(Collectors.toSet());
Sho SHIMIZUa09e1bb2016-08-01 14:25:25 -07001298 hostInterfaces.forEach(routerInf -> {
lishuai8798bbe2016-05-05 16:02:03 +08001299 Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces)
1300 .stream().filter(r -> r.tenantId().equals(tenantId))
1301 .filter(r -> r.routerId().equals(routerInf.routerId()))
1302 .collect(Collectors.toSet());
1303 long count = interfacesSet.stream()
1304 .filter(r -> !r.subnetId().equals(subnetId)).count();
1305 if (count > 0) {
1306 TenantRouter tenantRouter = TenantRouter
1307 .tenantRouter(routerInf.tenantId(), routerInf.routerId());
1308 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
1309 if (operation == Objective.Operation.ADD) {
1310 if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
1311 applyEastWestL3Flows(host, l3vni, operation);
1312 } else {
1313 if (interfacesSet.size() > 1) {
1314 programInterfacesSet(interfacesSet, operation);
1315 }
1316 }
1317 } else if (operation == Objective.Operation.REMOVE) {
1318 if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
1319 applyEastWestL3Flows(host, l3vni, operation);
lishuai858efd32015-12-04 14:30:36 +08001320 }
1321 }
lishuai858efd32015-12-04 14:30:36 +08001322 }
lishuai8798bbe2016-05-05 16:02:03 +08001323 });
lishuai858efd32015-12-04 14:30:36 +08001324 // L3 external and internal network access to each other
1325 FloatingIp floatingIp = null;
1326 Iterable<FloatingIp> floatingIps = floatingIpService.getFloatingIps();
1327 Set<FloatingIp> floatingIpSet = Sets.newHashSet(floatingIps).stream()
1328 .filter(f -> f.tenantId().equals(tenantId))
1329 .collect(Collectors.toSet());
1330 for (FloatingIp f : floatingIpSet) {
1331 IpAddress fixedIp = f.fixedIp();
lishuaibee5eca2016-05-10 19:17:37 +08001332 if (fixedIp != null && fixedIp.equals(hostIp)) {
lishuai858efd32015-12-04 14:30:36 +08001333 floatingIp = f;
1334 break;
1335 }
1336 }
1337 if (floatingIp != null) {
lishuai8798bbe2016-05-05 16:02:03 +08001338 TenantRouter tenantRouter = TenantRouter
1339 .tenantRouter(floatingIp.tenantId(), floatingIp.routerId());
1340 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
lishuai858efd32015-12-04 14:30:36 +08001341 VirtualPort fipPort = virtualPortService
1342 .getPort(floatingIp.networkId(), floatingIp.floatingIp());
1343 if (fipPort == null) {
1344 fipPort = VtnData.getPort(vPortStore, floatingIp.networkId(),
1345 floatingIp.floatingIp());
1346 }
lishuaibee5eca2016-05-10 19:17:37 +08001347 applyNorthSouthL3Flows(deviceId, true, tenantRouter, host, port,
1348 fipPort, floatingIp, l3vni, exPort,
1349 operation);
lishuai858efd32015-12-04 14:30:36 +08001350 }
1351 }
lishuai74f2d532015-12-10 17:03:34 +08001352
1353 public static void setExPortName(String name) {
lishuaib43dbf72016-01-06 11:11:35 +08001354 exPortMap.put(EX_PORT_KEY, name);
lishuai74f2d532015-12-10 17:03:34 +08001355 }
Bob zhou59a21062016-05-12 19:40:05 +08001356
1357 /**
1358 * Packet processor responsible for forwarding packets along their paths.
1359 */
1360 private class VtnL3PacketProcessor implements PacketProcessor {
1361
1362 @Override
1363 public void process(PacketContext context) {
1364 InboundPacket pkt = context.inPacket();
1365 ConnectPoint connectPoint = pkt.receivedFrom();
1366 DeviceId deviceId = connectPoint.deviceId();
1367 Ethernet ethPkt = pkt.parsed();
1368 if (ethPkt == null) {
1369 return;
1370 }
1371 if (ethPkt.getEtherType() == Ethernet.TYPE_ARP) {
1372 ARP arpPacket = (ARP) ethPkt.getPayload();
1373 if ((arpPacket.getOpCode() == ARP.OP_REQUEST)) {
1374 arprequestProcess(arpPacket, deviceId);
1375 } else if (arpPacket.getOpCode() == ARP.OP_REPLY) {
1376 arpresponceProcess(arpPacket, deviceId);
1377 }
1378 } else if (ethPkt.getEtherType() == Ethernet.TYPE_IPV4) {
1379 if (ethPkt.getDestinationMAC().isMulticast()) {
1380 return;
1381 }
1382 IPv4 ip = (IPv4) ethPkt.getPayload();
1383 upStreamPacketProcessor(ip, deviceId);
1384
1385 } else {
1386 return;
1387 }
1388 }
1389
1390 private void arprequestProcess(ARP arpPacket, DeviceId deviceId) {
1391 MacAddress dstMac = MacAddress
1392 .valueOf(arpPacket.getSenderHardwareAddress());
1393 IpAddress srcIp = IpAddress.valueOf(IPv4
1394 .toIPv4Address(arpPacket.getTargetProtocolAddress()));
1395 IpAddress dstIp = IpAddress.valueOf(IPv4
1396 .toIPv4Address(arpPacket.getSenderProtocolAddress()));
1397 FloatingIp floatingIp = floatingIpStore.get(srcIp);
1398 if (floatingIp == null) {
1399 return;
1400 }
1401 DeviceId deviceIdOfFloatingIp = getDeviceIdOfFloatingIP(floatingIp);
1402 if (!deviceId.equals(deviceIdOfFloatingIp)) {
1403 return;
1404 }
1405 Port exPort = exPortOfDevice.get(deviceId);
1406 MacAddress srcMac = MacAddress.valueOf(exPort.annotations()
1407 .value(AnnotationKeys.PORT_MAC));
1408 if (!downloadSnatRules(deviceId, srcMac, srcIp, dstMac, dstIp,
1409 floatingIp)) {
1410 return;
1411 }
1412 Ethernet ethernet = buildArpResponse(dstIp, dstMac, srcIp, srcMac);
1413 if (ethernet != null) {
1414 sendPacketOut(deviceId, exPort.number(), ethernet);
1415 }
1416 }
1417
1418 private void arpresponceProcess(ARP arpPacket, DeviceId deviceId) {
1419 MacAddress srcMac = MacAddress
1420 .valueOf(arpPacket.getTargetHardwareAddress());
1421 MacAddress dstMac = MacAddress
1422 .valueOf(arpPacket.getSenderHardwareAddress());
1423 IpAddress srcIp = IpAddress.valueOf(IPv4
1424 .toIPv4Address(arpPacket.getTargetProtocolAddress()));
1425 IpAddress dstIp = IpAddress.valueOf(IPv4
1426 .toIPv4Address(arpPacket.getSenderProtocolAddress()));
1427 FloatingIp floatingIp = floatingIpStore.get(srcIp);
1428 if (floatingIp == null) {
1429 return;
1430 }
1431 DeviceId deviceIdOfFloatingIp = getDeviceIdOfFloatingIP(floatingIp);
1432 if (!deviceId.equals(deviceIdOfFloatingIp)) {
1433 return;
1434 }
1435 if (!downloadSnatRules(deviceId, srcMac, srcIp, dstMac, dstIp,
1436 floatingIp)) {
1437 return;
1438 }
1439 }
1440
1441 private void upStreamPacketProcessor(IPv4 ipPacket, DeviceId deviceId) {
1442 IpAddress srcIp = IpAddress.valueOf(ipPacket.getSourceAddress());
1443 IpAddress dstIp = IpAddress.valueOf(ipPacket.getDestinationAddress());
1444 FloatingIp floatingIp = null;
1445 Collection<FloatingIp> floatingIps = floatingIpService
1446 .getFloatingIps();
1447 Set<FloatingIp> floatingIpSet = Sets.newHashSet(floatingIps)
1448 .stream().collect(Collectors.toSet());
1449 for (FloatingIp f : floatingIpSet) {
1450 IpAddress fixIp = f.fixedIp();
1451 if (fixIp != null && fixIp.equals(srcIp)) {
1452 floatingIp = f;
1453 break;
1454 }
1455 }
1456 if (floatingIp == null) {
1457 return;
1458 }
1459 Subnet subnet = getSubnetOfFloatingIP(floatingIp);
1460 IpAddress gwIp = subnet.gatewayIp();
1461 Port exportPort = exPortOfDevice.get(deviceId);
1462 MacAddress exPortMac = MacAddress.valueOf(exportPort.annotations()
1463 .value(AnnotationKeys.PORT_MAC));
1464 IpPrefix ipPrefix = subnet.cidr();
1465 if (ipPrefix == null) {
1466 return;
1467 }
1468 int mask = ipPrefix.prefixLength();
1469 if (mask <= 0) {
1470 return;
1471 }
1472 Ethernet ethernet = null;
1473 // if the same ip segment
1474 if (IpUtil.checkSameSegment(floatingIp.floatingIp(), dstIp, mask)) {
1475 ethernet = buildArpRequest(dstIp, floatingIp.floatingIp(),
1476 exPortMac);
1477 } else {
1478 ethernet = buildArpRequest(gwIp, floatingIp.floatingIp(),
1479 exPortMac);
1480 }
1481 if (ethernet != null) {
1482 sendPacketOut(deviceId, exportPort.number(), ethernet);
1483 }
1484 }
1485 }
1486
1487 private Ethernet buildArpRequest(IpAddress targetIp, IpAddress sourceIp,
1488 MacAddress sourceMac) {
1489 ARP arp = new ARP();
1490 arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
1491 .setHardwareAddressLength((byte) Ethernet.DATALAYER_ADDRESS_LENGTH)
1492 .setProtocolType(ARP.PROTO_TYPE_IP)
1493 .setProtocolAddressLength((byte) Ip4Address.BYTE_LENGTH)
1494 .setOpCode(ARP.OP_REQUEST);
1495
1496 arp.setSenderHardwareAddress(sourceMac.toBytes())
1497 .setSenderProtocolAddress(sourceIp.getIp4Address().toInt())
1498 .setTargetHardwareAddress(ZERO_MAC_ADDRESS)
1499 .setTargetProtocolAddress(targetIp.getIp4Address().toInt());
1500
1501 Ethernet ethernet = new Ethernet();
1502 ethernet.setEtherType(Ethernet.TYPE_ARP)
1503 .setDestinationMACAddress(MacAddress.BROADCAST)
1504 .setSourceMACAddress(sourceMac)
1505 .setPayload(arp);
1506
1507 ethernet.setPad(true);
1508 return ethernet;
1509 }
1510
1511 private Ethernet buildArpResponse(IpAddress targetIp, MacAddress targetMac,
1512 IpAddress sourceIp, MacAddress sourceMac) {
1513 ARP arp = new ARP();
1514 arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
1515 .setHardwareAddressLength((byte) Ethernet.DATALAYER_ADDRESS_LENGTH)
1516 .setProtocolType(ARP.PROTO_TYPE_IP)
1517 .setProtocolAddressLength((byte) Ip4Address.BYTE_LENGTH)
1518 .setOpCode(ARP.OP_REPLY);
1519
1520 arp.setSenderHardwareAddress(sourceMac.toBytes())
1521 .setSenderProtocolAddress(sourceIp.getIp4Address().toInt())
1522 .setTargetHardwareAddress(targetMac.toBytes())
1523 .setTargetProtocolAddress(targetIp.getIp4Address().toInt());
1524
1525 Ethernet ethernet = new Ethernet();
1526 ethernet.setEtherType(Ethernet.TYPE_ARP)
1527 .setDestinationMACAddress(targetMac)
1528 .setSourceMACAddress(sourceMac)
1529 .setPayload(arp);
1530
1531 ethernet.setPad(true);
1532
1533 return ethernet;
1534 }
1535
1536 private void sendPacketOut(DeviceId deviceId, PortNumber portNumber,
1537 Ethernet payload) {
1538 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
1539 .setOutput(portNumber).build();
1540 OutboundPacket packet = new DefaultOutboundPacket(deviceId, treatment,
1541 ByteBuffer
1542 .wrap(payload
1543 .serialize()));
1544 packetService.emit(packet);
1545 }
1546
1547 private Subnet getSubnetOfFloatingIP(FloatingIp floatingIp) {
1548 DeviceId exVmPortId = DeviceId
1549 .deviceId(floatingIp.id().floatingIpId().toString());
1550 Collection<VirtualPort> exVmPortList = virtualPortService
1551 .getPorts(exVmPortId);
1552 VirtualPort exVmPort = null;
1553 if (exVmPortList != null) {
1554 exVmPort = exVmPortList.iterator().next();
1555 }
1556 if (exVmPort == null) {
1557 return null;
1558 }
1559 Set<FixedIp> fixedIps = exVmPort.fixedIps();
1560 SubnetId subnetId = null;
1561 for (FixedIp f : fixedIps) {
1562 IpAddress fp = f.ip();
1563 if (fp.equals(floatingIp.floatingIp())) {
1564 subnetId = f.subnetId();
1565 break;
1566 }
1567 }
1568 if (subnetId == null) {
1569 return null;
1570 }
1571 Subnet subnet = subnetService.getSubnet(subnetId);
1572 return subnet;
1573 }
1574
1575 private DeviceId getDeviceIdOfFloatingIP(FloatingIp floatingIp) {
1576 VirtualPortId vmPortId = floatingIp.portId();
1577 VirtualPort vmPort = virtualPortService.getPort(vmPortId);
1578 if (vmPort == null) {
1579 vmPort = VtnData.getPort(vPortStore, vmPortId);
1580 }
1581 Set<Host> hostSet = hostService.getHostsByMac(vmPort.macAddress());
1582 Host host = null;
1583 for (Host h : hostSet) {
1584 String ifaceid = h.annotations().value(IFACEID);
1585 if (ifaceid != null && ifaceid.equals(vmPortId.portId())) {
1586 host = h;
1587 break;
1588 }
1589 }
1590 if (host == null) {
1591 return null;
1592 } else {
1593 return host.location().deviceId();
1594 }
1595 }
1596
1597 private boolean downloadSnatRules(DeviceId deviceId, MacAddress srcMac,
1598 IpAddress srcIp, MacAddress dstMac,
1599 IpAddress dstIp, FloatingIp floatingIp) {
1600 TenantNetwork exNetwork = tenantNetworkService
1601 .getNetwork(floatingIp.networkId());
1602 IpAddress fixedIp = floatingIp.fixedIp();
1603 VirtualPortId vmPortId = floatingIp.portId();
1604 VirtualPort vmPort = virtualPortService.getPort(vmPortId);
1605 if (vmPort == null) {
1606 vmPort = VtnData.getPort(vPortStore, vmPortId);
1607 }
1608 Subnet subnet = getSubnetOfFloatingIP(floatingIp);
1609 IpPrefix ipPrefix = subnet.cidr();
1610 IpAddress gwIp = subnet.gatewayIp();
1611 if (ipPrefix == null) {
1612 return false;
1613 }
1614 int mask = ipPrefix.prefixLength();
1615 if (mask <= 0) {
1616 return false;
1617 }
1618 TenantRouter tenantRouter = TenantRouter
1619 .tenantRouter(floatingIp.tenantId(), floatingIp.routerId());
1620 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
1621 // if the same ip segment
1622 if (IpUtil.checkSameSegment(srcIp, dstIp, mask)) {
1623 snatService.programSnatSameSegmentRules(deviceId, l3vni, fixedIp,
1624 dstIp, dstMac, srcMac,
1625 srcIp,
1626 exNetwork.segmentationId(),
1627 Objective.Operation.ADD);
1628 if (dstIp.equals(gwIp)) {
1629 snatService
1630 .programSnatDiffSegmentRules(deviceId, l3vni, fixedIp,
1631 dstMac, srcMac, srcIp,
1632 exNetwork.segmentationId(),
1633 Objective.Operation.ADD);
1634 }
1635 }
1636 return true;
1637 }
1638
1639 private void removeRulesInSnat(DeviceId deviceId, IpAddress fixedIp) {
1640 for (FlowEntry f : flowRuleService.getFlowEntries(deviceId)) {
1641 if (f.tableId() == SNAT_TABLE
1642 && f.priority() > SNAT_DEFAULT_RULE_PRIORITY) {
1643 String srcIp = f.selector()
1644 .getCriterion(Criterion.Type.IPV4_SRC).toString();
1645 int priority = f.priority();
1646 if (srcIp != null && srcIp.contains(fixedIp.toString())) {
1647 log.info("Match snat rules bob");
1648 TrafficSelector selector = f.selector();
1649 TrafficTreatment treatment = f.treatment();
1650 snatService.removeSnatRules(deviceId, selector, treatment,
1651 priority,
1652 Objective.Operation.REMOVE);
1653
1654 }
1655 }
1656 }
1657 }
lishuai6c56f5e2015-11-17 16:38:19 +08001658}