blob: 8f46dbea7f3b6d337a5090a88c1c414089f24490 [file] [log] [blame]
lishuai6c56f5e2015-11-17 16:38:19 +08001/*
Brian O'Connor5ab426f2016-04-09 01:19:45 -07002 * Copyright 2015-present Open Networking Laboratory
lishuai6c56f5e2015-11-17 16:38:19 +08003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package org.onosproject.vtn.manager.impl;
17
jiangruic69a7fd2015-11-19 15:40:01 +080018import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST;
lishuai6c56f5e2015-11-17 16:38:19 +080019import static org.slf4j.LoggerFactory.getLogger;
20
Bob zhou59a21062016-05-12 19:40:05 +080021import java.nio.ByteBuffer;
lishuai858efd32015-12-04 14:30:36 +080022import java.util.ArrayList;
lishuaib43dbf72016-01-06 11:11:35 +080023import java.util.Arrays;
lishuai6c56f5e2015-11-17 16:38:19 +080024import java.util.Collection;
25import java.util.HashMap;
26import java.util.HashSet;
27import java.util.Iterator;
jiangruic69a7fd2015-11-19 15:40:01 +080028import java.util.List;
lishuai6c56f5e2015-11-17 16:38:19 +080029import java.util.Map;
30import java.util.Set;
Bob zhou59a21062016-05-12 19:40:05 +080031import java.util.UUID;
lishuai74f2d532015-12-10 17:03:34 +080032import java.util.concurrent.ConcurrentHashMap;
lishuai858efd32015-12-04 14:30:36 +080033import java.util.stream.Collectors;
lishuai6c56f5e2015-11-17 16:38:19 +080034
35import org.apache.felix.scr.annotations.Activate;
36import org.apache.felix.scr.annotations.Component;
37import org.apache.felix.scr.annotations.Deactivate;
38import org.apache.felix.scr.annotations.Reference;
39import org.apache.felix.scr.annotations.ReferenceCardinality;
40import org.apache.felix.scr.annotations.Service;
Bob zhou59a21062016-05-12 19:40:05 +080041import org.onlab.packet.ARP;
42import org.onlab.packet.Ethernet;
43import org.onlab.packet.IPv4;
jiangruic69a7fd2015-11-19 15:40:01 +080044import org.onlab.packet.Ip4Address;
lishuai6c56f5e2015-11-17 16:38:19 +080045import org.onlab.packet.IpAddress;
Bob zhou59a21062016-05-12 19:40:05 +080046import org.onlab.packet.IpPrefix;
lishuai6c56f5e2015-11-17 16:38:19 +080047import org.onlab.packet.MacAddress;
48import org.onlab.util.KryoNamespace;
49import org.onosproject.core.ApplicationId;
50import org.onosproject.core.CoreService;
51import org.onosproject.mastership.MastershipService;
lishuai858efd32015-12-04 14:30:36 +080052import org.onosproject.net.AnnotationKeys;
Bob zhou59a21062016-05-12 19:40:05 +080053import org.onosproject.net.ConnectPoint;
lishuai6c56f5e2015-11-17 16:38:19 +080054import org.onosproject.net.Device;
55import org.onosproject.net.DeviceId;
56import org.onosproject.net.Host;
lishuai858efd32015-12-04 14:30:36 +080057import org.onosproject.net.HostId;
lishuai6c56f5e2015-11-17 16:38:19 +080058import org.onosproject.net.Port;
59import org.onosproject.net.PortNumber;
60import org.onosproject.net.behaviour.BridgeConfig;
61import org.onosproject.net.behaviour.BridgeDescription;
jiangruic69a7fd2015-11-19 15:40:01 +080062import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
lishuai6c56f5e2015-11-17 16:38:19 +080063import org.onosproject.net.config.NetworkConfigService;
64import org.onosproject.net.config.basics.BasicDeviceConfig;
65import org.onosproject.net.device.DeviceEvent;
66import org.onosproject.net.device.DeviceListener;
67import org.onosproject.net.device.DeviceService;
68import org.onosproject.net.driver.DriverHandler;
69import org.onosproject.net.driver.DriverService;
jiangruic69a7fd2015-11-19 15:40:01 +080070import org.onosproject.net.flow.DefaultTrafficTreatment;
Bob zhou59a21062016-05-12 19:40:05 +080071import org.onosproject.net.flow.FlowEntry;
72import org.onosproject.net.flow.FlowRuleService;
73import org.onosproject.net.flow.TrafficSelector;
74import org.onosproject.net.flow.TrafficTreatment;
jiangruic69a7fd2015-11-19 15:40:01 +080075import org.onosproject.net.flow.TrafficTreatment.Builder;
Bob zhou59a21062016-05-12 19:40:05 +080076import org.onosproject.net.flow.criteria.Criterion;
jiangruic69a7fd2015-11-19 15:40:01 +080077import org.onosproject.net.flow.instructions.ExtensionTreatment;
lishuai6c56f5e2015-11-17 16:38:19 +080078import org.onosproject.net.flowobjective.Objective;
jiangruic69a7fd2015-11-19 15:40:01 +080079import org.onosproject.net.group.DefaultGroupBucket;
80import org.onosproject.net.group.DefaultGroupDescription;
81import org.onosproject.net.group.DefaultGroupKey;
82import org.onosproject.net.group.GroupBucket;
83import org.onosproject.net.group.GroupBuckets;
84import org.onosproject.net.group.GroupDescription;
85import org.onosproject.net.group.GroupKey;
86import org.onosproject.net.group.GroupService;
lishuai6c56f5e2015-11-17 16:38:19 +080087import org.onosproject.net.host.HostEvent;
88import org.onosproject.net.host.HostListener;
89import org.onosproject.net.host.HostService;
Bob zhou59a21062016-05-12 19:40:05 +080090import org.onosproject.net.packet.DefaultOutboundPacket;
91import org.onosproject.net.packet.InboundPacket;
92import org.onosproject.net.packet.OutboundPacket;
93import org.onosproject.net.packet.PacketContext;
94import org.onosproject.net.packet.PacketProcessor;
95import org.onosproject.net.packet.PacketService;
lishuai6c56f5e2015-11-17 16:38:19 +080096import org.onosproject.store.serializers.KryoNamespaces;
lishuaib43dbf72016-01-06 11:11:35 +080097import org.onosproject.store.service.ConsistentMap;
lishuai6c56f5e2015-11-17 16:38:19 +080098import org.onosproject.store.service.EventuallyConsistentMap;
99import org.onosproject.store.service.LogicalClockService;
lishuaib43dbf72016-01-06 11:11:35 +0800100import org.onosproject.store.service.Serializer;
lishuai6c56f5e2015-11-17 16:38:19 +0800101import org.onosproject.store.service.StorageService;
lishuaib43dbf72016-01-06 11:11:35 +0800102import org.onosproject.store.service.Versioned;
Jonathan Hart51539b82015-10-29 09:53:04 -0700103import org.onosproject.vtn.manager.VtnService;
lishuai858efd32015-12-04 14:30:36 +0800104import org.onosproject.vtn.table.ArpService;
lishuai6c56f5e2015-11-17 16:38:19 +0800105import org.onosproject.vtn.table.ClassifierService;
lishuai858efd32015-12-04 14:30:36 +0800106import org.onosproject.vtn.table.DnatService;
lishuai6c56f5e2015-11-17 16:38:19 +0800107import org.onosproject.vtn.table.L2ForwardService;
lishuai858efd32015-12-04 14:30:36 +0800108import org.onosproject.vtn.table.L3ForwardService;
109import org.onosproject.vtn.table.SnatService;
lishuai74f2d532015-12-10 17:03:34 +0800110import org.onosproject.vtn.table.impl.ArpServiceImpl;
lishuai6c56f5e2015-11-17 16:38:19 +0800111import org.onosproject.vtn.table.impl.ClassifierServiceImpl;
lishuai74f2d532015-12-10 17:03:34 +0800112import org.onosproject.vtn.table.impl.DnatServiceImpl;
lishuai6c56f5e2015-11-17 16:38:19 +0800113import org.onosproject.vtn.table.impl.L2ForwardServiceImpl;
lishuai74f2d532015-12-10 17:03:34 +0800114import org.onosproject.vtn.table.impl.L3ForwardServiceImpl;
115import org.onosproject.vtn.table.impl.SnatServiceImpl;
lishuai6c56f5e2015-11-17 16:38:19 +0800116import org.onosproject.vtn.util.DataPathIdGenerator;
Bob zhou59a21062016-05-12 19:40:05 +0800117import org.onosproject.vtn.util.IpUtil;
lishuai6c56f5e2015-11-17 16:38:19 +0800118import org.onosproject.vtn.util.VtnConfig;
119import org.onosproject.vtn.util.VtnData;
lishuai7dc63d92015-11-27 17:15:25 +0800120import org.onosproject.vtnrsc.AllowedAddressPair;
121import org.onosproject.vtnrsc.BindingHostId;
Bob zhou59a21062016-05-12 19:40:05 +0800122import org.onosproject.vtnrsc.DefaultFloatingIp;
lishuai7dc63d92015-11-27 17:15:25 +0800123import org.onosproject.vtnrsc.DefaultVirtualPort;
124import org.onosproject.vtnrsc.FixedIp;
lishuai858efd32015-12-04 14:30:36 +0800125import org.onosproject.vtnrsc.FloatingIp;
Bob zhou59a21062016-05-12 19:40:05 +0800126import org.onosproject.vtnrsc.FloatingIpId;
lishuaibee5eca2016-05-10 19:17:37 +0800127import org.onosproject.vtnrsc.RouterId;
lishuai858efd32015-12-04 14:30:36 +0800128import org.onosproject.vtnrsc.RouterInterface;
lishuai7dc63d92015-11-27 17:15:25 +0800129import org.onosproject.vtnrsc.SecurityGroup;
lishuai6c56f5e2015-11-17 16:38:19 +0800130import org.onosproject.vtnrsc.SegmentationId;
Wu wenbinacc10ea2016-05-06 16:48:56 +0800131import org.onosproject.vtnrsc.Subnet;
lishuai6c56f5e2015-11-17 16:38:19 +0800132import org.onosproject.vtnrsc.SubnetId;
133import org.onosproject.vtnrsc.TenantId;
134import org.onosproject.vtnrsc.TenantNetwork;
135import org.onosproject.vtnrsc.TenantNetworkId;
lishuai8798bbe2016-05-05 16:02:03 +0800136import org.onosproject.vtnrsc.TenantRouter;
lishuai6c56f5e2015-11-17 16:38:19 +0800137import org.onosproject.vtnrsc.VirtualPort;
138import org.onosproject.vtnrsc.VirtualPortId;
lishuai858efd32015-12-04 14:30:36 +0800139import org.onosproject.vtnrsc.event.VtnRscEvent;
140import org.onosproject.vtnrsc.event.VtnRscEventFeedback;
141import org.onosproject.vtnrsc.event.VtnRscListener;
142import org.onosproject.vtnrsc.floatingip.FloatingIpService;
lishuai858efd32015-12-04 14:30:36 +0800143import org.onosproject.vtnrsc.routerinterface.RouterInterfaceService;
144import org.onosproject.vtnrsc.service.VtnRscService;
145import org.onosproject.vtnrsc.subnet.SubnetService;
lishuai6c56f5e2015-11-17 16:38:19 +0800146import org.onosproject.vtnrsc.tenantnetwork.TenantNetworkService;
147import org.onosproject.vtnrsc.virtualport.VirtualPortService;
148import org.slf4j.Logger;
149
jiangruic69a7fd2015-11-19 15:40:01 +0800150import com.google.common.collect.Lists;
lishuai6c56f5e2015-11-17 16:38:19 +0800151import com.google.common.collect.Sets;
152
153/**
154 * Provides implementation of VTNService.
155 */
156@Component(immediate = true)
157@Service
Jonathan Hart51539b82015-10-29 09:53:04 -0700158public class VtnManager implements VtnService {
lishuai6c56f5e2015-11-17 16:38:19 +0800159 private final Logger log = getLogger(getClass());
160 private static final String APP_ID = "org.onosproject.app.vtn";
161
162 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
163 protected NetworkConfigService configService;
164
165 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
166 protected DeviceService deviceService;
167
168 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
169 protected HostService hostService;
170
171 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
172 protected CoreService coreService;
173
174 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
175 protected StorageService storageService;
176
177 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
178 protected TenantNetworkService tenantNetworkService;
179
180 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
181 protected VirtualPortService virtualPortService;
182
183 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
184 protected DriverService driverService;
185
186 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
187 protected LogicalClockService clockService;
188
189 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
190 protected MastershipService mastershipService;
191
jiangruic69a7fd2015-11-19 15:40:01 +0800192 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
193 protected GroupService groupService;
194
lishuai858efd32015-12-04 14:30:36 +0800195 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
196 protected SubnetService subnetService;
197
198 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
199 protected VtnRscService vtnRscService;
200
201 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
202 protected FloatingIpService floatingIpService;
203
204 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
lishuai858efd32015-12-04 14:30:36 +0800205 protected RouterInterfaceService routerInterfaceService;
206
Bob zhou59a21062016-05-12 19:40:05 +0800207 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
208 protected FlowRuleService flowRuleService;
209
lishuai6c56f5e2015-11-17 16:38:19 +0800210 private ApplicationId appId;
211 private ClassifierService classifierService;
212 private L2ForwardService l2ForwardService;
lishuai858efd32015-12-04 14:30:36 +0800213 private ArpService arpService;
214 private L3ForwardService l3ForwardService;
215 private SnatService snatService;
216 private DnatService dnatService;
lishuai6c56f5e2015-11-17 16:38:19 +0800217
218 private final HostListener hostListener = new InnerHostListener();
219 private final DeviceListener deviceListener = new InnerDeviceListener();
lishuai858efd32015-12-04 14:30:36 +0800220 private final VtnRscListener l3EventListener = new VtnL3EventListener();
lishuai6c56f5e2015-11-17 16:38:19 +0800221
lishuaib43dbf72016-01-06 11:11:35 +0800222 private static final String EX_PORT_KEY = "exPortKey";
lishuai6c56f5e2015-11-17 16:38:19 +0800223 private static final String IFACEID = "ifaceid";
224 private static final String CONTROLLER_IP_KEY = "ipaddress";
225 public static final String DRIVER_NAME = "onosfw";
lishuai7dc63d92015-11-27 17:15:25 +0800226 private static final String VIRTUALPORT = "vtn-virtual-port";
lishuai6c56f5e2015-11-17 16:38:19 +0800227 private static final String SWITCHES_OF_CONTROLLER = "switchesOfController";
228 private static final String SWITCH_OF_LOCAL_HOST_PORTS = "switchOfLocalHostPorts";
lishuai8798bbe2016-05-05 16:02:03 +0800229 private static final String ROUTERINF_FLAG_OF_TENANTROUTER = "routerInfFlagOfTenantRouter";
lishuai858efd32015-12-04 14:30:36 +0800230 private static final String HOSTS_OF_SUBNET = "hostsOfSubnet";
231 private static final String EX_PORT_OF_DEVICE = "exPortOfDevice";
lishuaib43dbf72016-01-06 11:11:35 +0800232 private static final String EX_PORT_MAP = "exPortMap";
jiangruic69a7fd2015-11-19 15:40:01 +0800233 private static final String DEFAULT_IP = "0.0.0.0";
Bob zhou59a21062016-05-12 19:40:05 +0800234 private static final String FLOATINGSTORE = "vtn-floatingIp";
Wu wenbinacc10ea2016-05-06 16:48:56 +0800235 private static final String USERDATA_IP = "169.254.169.254";
lishuai858efd32015-12-04 14:30:36 +0800236 private static final int SUBNET_NUM = 2;
Bob zhou59a21062016-05-12 19:40:05 +0800237 private static final int SNAT_TABLE = 40;
238 private static final int SNAT_DEFAULT_RULE_PRIORITY = 0;
239 private static final byte[] ZERO_MAC_ADDRESS = MacAddress.ZERO.toBytes();
lishuai6c56f5e2015-11-17 16:38:19 +0800240
lishuai7dc63d92015-11-27 17:15:25 +0800241 private EventuallyConsistentMap<VirtualPortId, VirtualPort> vPortStore;
lishuai6c56f5e2015-11-17 16:38:19 +0800242 private EventuallyConsistentMap<IpAddress, Boolean> switchesOfController;
jiangruic69a7fd2015-11-19 15:40:01 +0800243 private EventuallyConsistentMap<DeviceId, NetworkOfLocalHostPorts> switchOfLocalHostPorts;
lishuai858efd32015-12-04 14:30:36 +0800244 private EventuallyConsistentMap<SubnetId, Map<HostId, Host>> hostsOfSubnet;
lishuai8798bbe2016-05-05 16:02:03 +0800245 private EventuallyConsistentMap<TenantRouter, Boolean> routerInfFlagOfTenantRouter;
lishuai858efd32015-12-04 14:30:36 +0800246 private EventuallyConsistentMap<DeviceId, Port> exPortOfDevice;
Bob zhou59a21062016-05-12 19:40:05 +0800247 private EventuallyConsistentMap<IpAddress, FloatingIp> floatingIpStore;
lishuaib43dbf72016-01-06 11:11:35 +0800248 private static ConsistentMap<String, String> exPortMap;
lishuai6c56f5e2015-11-17 16:38:19 +0800249
Bob zhou59a21062016-05-12 19:40:05 +0800250 private VtnL3PacketProcessor l3PacketProcessor = new VtnL3PacketProcessor();
251 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
252 protected PacketService packetService;
253
lishuai6c56f5e2015-11-17 16:38:19 +0800254 @Activate
255 public void activate() {
256 appId = coreService.registerApplication(APP_ID);
257 classifierService = new ClassifierServiceImpl(appId);
258 l2ForwardService = new L2ForwardServiceImpl(appId);
lishuai74f2d532015-12-10 17:03:34 +0800259 arpService = new ArpServiceImpl(appId);
260 l3ForwardService = new L3ForwardServiceImpl(appId);
261 snatService = new SnatServiceImpl(appId);
262 dnatService = new DnatServiceImpl(appId);
lishuai6c56f5e2015-11-17 16:38:19 +0800263
264 deviceService.addListener(deviceListener);
265 hostService.addListener(hostListener);
lishuai74f2d532015-12-10 17:03:34 +0800266 vtnRscService.addListener(l3EventListener);
lishuai6c56f5e2015-11-17 16:38:19 +0800267
268 KryoNamespace.Builder serializer = KryoNamespace.newBuilder()
269 .register(KryoNamespaces.API)
270 .register(NetworkOfLocalHostPorts.class)
271 .register(TenantNetworkId.class)
272 .register(Host.class)
273 .register(TenantNetwork.class)
Bob zhou59a21062016-05-12 19:40:05 +0800274 .register(TenantNetworkId.class)
lishuai6c56f5e2015-11-17 16:38:19 +0800275 .register(TenantId.class)
lishuai7dc63d92015-11-27 17:15:25 +0800276 .register(SubnetId.class)
277 .register(VirtualPortId.class)
278 .register(VirtualPort.State.class)
279 .register(AllowedAddressPair.class)
280 .register(FixedIp.class)
Bob zhou59a21062016-05-12 19:40:05 +0800281 .register(FloatingIp.class)
282 .register(FloatingIpId.class)
283 .register(FloatingIp.Status.class)
284 .register(UUID.class)
285 .register(DefaultFloatingIp.class)
lishuai7dc63d92015-11-27 17:15:25 +0800286 .register(BindingHostId.class)
287 .register(SecurityGroup.class)
288 .register(IpAddress.class)
lishuaibee5eca2016-05-10 19:17:37 +0800289 .register(DefaultVirtualPort.class)
290 .register(RouterId.class)
291 .register(TenantRouter.class);
Bob zhou59a21062016-05-12 19:40:05 +0800292 floatingIpStore = storageService
293 .<IpAddress, FloatingIp>eventuallyConsistentMapBuilder()
294 .withName(FLOATINGSTORE).withSerializer(serializer)
295 .withTimestampProvider((k, v) -> clockService.getTimestamp())
296 .build();
lishuai7dc63d92015-11-27 17:15:25 +0800297
298 vPortStore = storageService
299 .<VirtualPortId, VirtualPort>eventuallyConsistentMapBuilder()
300 .withName(VIRTUALPORT).withSerializer(serializer)
301 .withTimestampProvider((k, v) -> clockService.getTimestamp())
302 .build();
lishuai6c56f5e2015-11-17 16:38:19 +0800303
304 switchesOfController = storageService
305 .<IpAddress, Boolean>eventuallyConsistentMapBuilder()
306 .withName(SWITCHES_OF_CONTROLLER).withSerializer(serializer)
307 .withTimestampProvider((k, v) -> clockService.getTimestamp())
308 .build();
309
310 switchOfLocalHostPorts = storageService
jiangruic69a7fd2015-11-19 15:40:01 +0800311 .<DeviceId, NetworkOfLocalHostPorts>eventuallyConsistentMapBuilder()
312 .withName(SWITCH_OF_LOCAL_HOST_PORTS).withSerializer(serializer)
313 .withTimestampProvider((k, v) -> clockService.getTimestamp())
lishuai6c56f5e2015-11-17 16:38:19 +0800314 .build();
315
lishuai858efd32015-12-04 14:30:36 +0800316 hostsOfSubnet = storageService
317 .<SubnetId, Map<HostId, Host>>eventuallyConsistentMapBuilder()
318 .withName(HOSTS_OF_SUBNET).withSerializer(serializer)
319 .withTimestampProvider((k, v) -> clockService.getTimestamp())
320 .build();
321
lishuai8798bbe2016-05-05 16:02:03 +0800322 routerInfFlagOfTenantRouter = storageService
323 .<TenantRouter, Boolean>eventuallyConsistentMapBuilder()
324 .withName(ROUTERINF_FLAG_OF_TENANTROUTER).withSerializer(serializer)
lishuai858efd32015-12-04 14:30:36 +0800325 .withTimestampProvider((k, v) -> clockService.getTimestamp())
326 .build();
327
328 exPortOfDevice = storageService
329 .<DeviceId, Port>eventuallyConsistentMapBuilder()
330 .withName(EX_PORT_OF_DEVICE).withSerializer(serializer)
331 .withTimestampProvider((k, v) -> clockService.getTimestamp())
332 .build();
333
lishuaib43dbf72016-01-06 11:11:35 +0800334 exPortMap = storageService
335 .<String, String>consistentMapBuilder()
336 .withName(EX_PORT_MAP)
337 .withApplicationId(appId)
338 .withPurgeOnUninstall()
339 .withSerializer(Serializer.using(Arrays.asList(KryoNamespaces.API)))
340 .build();
341
Bob zhou59a21062016-05-12 19:40:05 +0800342 packetService.addProcessor(l3PacketProcessor, PacketProcessor.director(0));
lishuai6c56f5e2015-11-17 16:38:19 +0800343 log.info("Started");
344 }
345
346 @Deactivate
347 public void deactivate() {
348 deviceService.removeListener(deviceListener);
349 hostService.removeListener(hostListener);
lishuai858efd32015-12-04 14:30:36 +0800350 vtnRscService.removeListener(l3EventListener);
lishuai6c56f5e2015-11-17 16:38:19 +0800351 log.info("Stopped");
352 }
353
354 @Override
355 public void onControllerDetected(Device controllerDevice) {
356 if (controllerDevice == null) {
357 log.error("The controller device is null");
358 return;
359 }
360 String localIpAddress = controllerDevice.annotations()
361 .value(CONTROLLER_IP_KEY);
362 IpAddress localIp = IpAddress.valueOf(localIpAddress);
363 DeviceId controllerDeviceId = controllerDevice.id();
364 DriverHandler handler = driverService.createHandler(controllerDeviceId);
365 if (mastershipService.isLocalMaster(controllerDeviceId)) {
366 // Get DataPathIdGenerator
367 String ipaddress = controllerDevice.annotations().value("ipaddress");
368 DataPathIdGenerator dpidGenerator = DataPathIdGenerator.builder()
369 .addIpAddress(ipaddress).build();
370 DeviceId deviceId = dpidGenerator.getDeviceId();
371 String dpid = dpidGenerator.getDpId();
372 // Inject pipeline driver name
373 BasicDeviceConfig config = configService.addConfig(deviceId,
374 BasicDeviceConfig.class);
375 config.driver(DRIVER_NAME);
376 configService.applyConfig(deviceId, BasicDeviceConfig.class, config.node());
377 // Add Bridge
lishuaib43dbf72016-01-06 11:11:35 +0800378 Versioned<String> exPortVersioned = exPortMap.get(EX_PORT_KEY);
379 if (exPortVersioned != null) {
380 VtnConfig.applyBridgeConfig(handler, dpid, exPortVersioned.value());
381 log.info("A new ovs is created in node {}", localIp.toString());
382 }
lishuai6c56f5e2015-11-17 16:38:19 +0800383 switchesOfController.put(localIp, true);
384 }
385 // Create tunnel in br-int on all controllers
386 programTunnelConfig(controllerDeviceId, localIp, handler);
387 }
388
389 @Override
390 public void onControllerVanished(Device controllerDevice) {
391 if (controllerDevice == null) {
392 log.error("The device is null");
393 return;
394 }
395 String dstIp = controllerDevice.annotations().value(CONTROLLER_IP_KEY);
396 IpAddress dstIpAddress = IpAddress.valueOf(dstIp);
397 DeviceId controllerDeviceId = controllerDevice.id();
398 if (mastershipService.isLocalMaster(controllerDeviceId)) {
399 switchesOfController.remove(dstIpAddress);
400 }
lishuai6c56f5e2015-11-17 16:38:19 +0800401 }
402
403 @Override
404 public void onOvsDetected(Device device) {
lishuaibee5eca2016-05-10 19:17:37 +0800405 if (device == null) {
406 log.error("The device is null");
407 return;
408 }
409 if (!mastershipService.isLocalMaster(device.id())) {
410 return;
411 }
lishuai6c56f5e2015-11-17 16:38:19 +0800412 // Create tunnel out flow rules
413 applyTunnelOut(device, Objective.Operation.ADD);
lishuaibee5eca2016-05-10 19:17:37 +0800414 // apply L3 arp flows
415 Iterable<RouterInterface> interfaces = routerInterfaceService
416 .getRouterInterfaces();
417 interfaces.forEach(routerInf -> {
418 VirtualPort gwPort = virtualPortService.getPort(routerInf.portId());
419 if (gwPort == null) {
420 gwPort = VtnData.getPort(vPortStore, routerInf.portId());
421 }
422 applyL3ArpFlows(device.id(), gwPort, Objective.Operation.ADD);
423 });
lishuai6c56f5e2015-11-17 16:38:19 +0800424 }
425
426 @Override
427 public void onOvsVanished(Device device) {
lishuaibee5eca2016-05-10 19:17:37 +0800428 if (device == null) {
429 log.error("The device is null");
430 return;
431 }
432 if (!mastershipService.isLocalMaster(device.id())) {
433 return;
434 }
lishuai6c56f5e2015-11-17 16:38:19 +0800435 // Remove Tunnel out flow rules
436 applyTunnelOut(device, Objective.Operation.REMOVE);
lishuaibee5eca2016-05-10 19:17:37 +0800437 // apply L3 arp flows
438 Iterable<RouterInterface> interfaces = routerInterfaceService
439 .getRouterInterfaces();
440 interfaces.forEach(routerInf -> {
441 VirtualPort gwPort = virtualPortService.getPort(routerInf.portId());
442 if (gwPort == null) {
443 gwPort = VtnData.getPort(vPortStore, routerInf.portId());
444 }
445 applyL3ArpFlows(device.id(), gwPort, Objective.Operation.REMOVE);
446 });
lishuai6c56f5e2015-11-17 16:38:19 +0800447 }
448
449 @Override
450 public void onHostDetected(Host host) {
lishuai858efd32015-12-04 14:30:36 +0800451 DeviceId deviceId = host.location().deviceId();
452 if (!mastershipService.isLocalMaster(deviceId)) {
453 return;
454 }
455 String ifaceId = host.annotations().value(IFACEID);
456 if (ifaceId == null) {
457 log.error("The ifaceId of Host is null");
458 return;
459 }
lishuaib43dbf72016-01-06 11:11:35 +0800460 programSffAndClassifierHost(host, Objective.Operation.ADD);
lishuai6c56f5e2015-11-17 16:38:19 +0800461 // apply L2 openflow rules
462 applyHostMonitoredL2Rules(host, Objective.Operation.ADD);
lishuai858efd32015-12-04 14:30:36 +0800463 // apply L3 openflow rules
464 applyHostMonitoredL3Rules(host, Objective.Operation.ADD);
lishuai6c56f5e2015-11-17 16:38:19 +0800465 }
466
467 @Override
468 public void onHostVanished(Host host) {
lishuai858efd32015-12-04 14:30:36 +0800469 DeviceId deviceId = host.location().deviceId();
470 if (!mastershipService.isLocalMaster(deviceId)) {
471 return;
472 }
473 String ifaceId = host.annotations().value(IFACEID);
474 if (ifaceId == null) {
475 log.error("The ifaceId of Host is null");
476 return;
477 }
lishuaib43dbf72016-01-06 11:11:35 +0800478 programSffAndClassifierHost(host, Objective.Operation.REMOVE);
lishuai6c56f5e2015-11-17 16:38:19 +0800479 // apply L2 openflow rules
480 applyHostMonitoredL2Rules(host, Objective.Operation.REMOVE);
lishuai858efd32015-12-04 14:30:36 +0800481 // apply L3 openflow rules
482 applyHostMonitoredL3Rules(host, Objective.Operation.REMOVE);
lishuai590d93a2015-12-11 13:05:14 +0800483 VirtualPortId virtualPortId = VirtualPortId.portId(ifaceId);
484 vPortStore.remove(virtualPortId);
lishuai6c56f5e2015-11-17 16:38:19 +0800485 }
486
487 private void programTunnelConfig(DeviceId localDeviceId, IpAddress localIp,
488 DriverHandler localHandler) {
jiangruic69a7fd2015-11-19 15:40:01 +0800489 if (mastershipService.isLocalMaster(localDeviceId)) {
490 VtnConfig.applyTunnelConfig(localHandler, localIp, IpAddress.valueOf(DEFAULT_IP));
491 log.info("Add tunnel on {}", localIp);
492 }
lishuai6c56f5e2015-11-17 16:38:19 +0800493 }
494
495 private void applyTunnelOut(Device device, Objective.Operation type) {
lishuai6c56f5e2015-11-17 16:38:19 +0800496 String controllerIp = VtnData.getControllerIpOfSwitch(device);
497 if (controllerIp == null) {
498 log.error("Can't find controller of device: {}",
499 device.id().toString());
500 return;
501 }
502 IpAddress ipAddress = IpAddress.valueOf(controllerIp);
503 if (!switchesOfController.containsKey(ipAddress)) {
504 log.error("Can't find controller of device: {}",
505 device.id().toString());
506 return;
507 }
508 if (type == Objective.Operation.ADD) {
lishuaia3e32342015-12-07 10:59:17 +0800509 // Save external port
510 Port export = getExPort(device.id());
lishuaid6f0c9e2015-12-16 11:40:01 +0800511 if (export != null) {
Bob zhou59a21062016-05-12 19:40:05 +0800512 classifierService.programExportPortArpClassifierRules(export,
513 device.id(),
514 type);
lishuaid6f0c9e2015-12-16 11:40:01 +0800515 exPortOfDevice.put(device.id(), export);
516 }
lishuai6c56f5e2015-11-17 16:38:19 +0800517 switchOfLocalHostPorts.put(device.id(), new NetworkOfLocalHostPorts());
518 } else if (type == Objective.Operation.REMOVE) {
lishuaia3e32342015-12-07 10:59:17 +0800519 exPortOfDevice.remove(device.id());
lishuai6c56f5e2015-11-17 16:38:19 +0800520 switchOfLocalHostPorts.remove(device.id());
521 }
522 Iterable<Device> devices = deviceService.getAvailableDevices();
523 DeviceId localControllerId = VtnData.getControllerId(device, devices);
524 DriverHandler handler = driverService.createHandler(localControllerId);
525 Set<PortNumber> ports = VtnConfig.getPortNumbers(handler);
526 Iterable<Host> allHosts = hostService.getHosts();
jiangruic69a7fd2015-11-19 15:40:01 +0800527 String tunnelName = "vxlan-" + DEFAULT_IP;
lishuai6c56f5e2015-11-17 16:38:19 +0800528 if (allHosts != null) {
529 Sets.newHashSet(allHosts).stream().forEach(host -> {
530 MacAddress hostMac = host.mac();
531 String ifaceId = host.annotations().value(IFACEID);
532 if (ifaceId == null) {
533 log.error("The ifaceId of Host is null");
534 return;
535 }
536 VirtualPortId virtualPortId = VirtualPortId.portId(ifaceId);
537 VirtualPort virtualPort = virtualPortService
538 .getPort(virtualPortId);
539 TenantNetwork network = tenantNetworkService
540 .getNetwork(virtualPort.networkId());
541 SegmentationId segmentationId = network.segmentationId();
542 DeviceId remoteDeviceId = host.location().deviceId();
543 Device remoteDevice = deviceService.getDevice(remoteDeviceId);
544 String remoteControllerIp = VtnData
545 .getControllerIpOfSwitch(remoteDevice);
546 if (remoteControllerIp == null) {
547 log.error("Can't find remote controller of device: {}",
548 remoteDeviceId.toString());
549 return;
550 }
551 IpAddress remoteIpAddress = IpAddress
552 .valueOf(remoteControllerIp);
lishuai6c56f5e2015-11-17 16:38:19 +0800553 ports.stream()
554 .filter(p -> p.name().equalsIgnoreCase(tunnelName))
555 .forEach(p -> {
556 l2ForwardService
557 .programTunnelOut(device.id(), segmentationId, p,
jiangruic69a7fd2015-11-19 15:40:01 +0800558 hostMac, type, remoteIpAddress);
lishuai6c56f5e2015-11-17 16:38:19 +0800559 });
560 });
561 }
562 }
563
lishuaib43dbf72016-01-06 11:11:35 +0800564 private void programSffAndClassifierHost(Host host, Objective.Operation type) {
565 DeviceId deviceId = host.location().deviceId();
566 String ifaceId = host.annotations().value(IFACEID);
567 VirtualPortId virtualPortId = VirtualPortId.portId(ifaceId);
568 VirtualPort virtualPort = virtualPortService.getPort(virtualPortId);
569 if (virtualPort == null) {
570 virtualPort = VtnData.getPort(vPortStore, virtualPortId);
571 }
572 TenantId tenantId = virtualPort.tenantId();
573 if (Objective.Operation.ADD == type) {
574 vtnRscService.addDeviceIdOfOvsMap(virtualPortId, tenantId, deviceId);
575 } else if (Objective.Operation.REMOVE == type) {
576 vtnRscService.removeDeviceIdOfOvsMap(host, tenantId, deviceId);
577 }
578 }
579
lishuai6c56f5e2015-11-17 16:38:19 +0800580 private void applyHostMonitoredL2Rules(Host host, Objective.Operation type) {
581 DeviceId deviceId = host.location().deviceId();
582 if (!mastershipService.isLocalMaster(deviceId)) {
583 return;
584 }
585 String ifaceId = host.annotations().value(IFACEID);
586 if (ifaceId == null) {
587 log.error("The ifaceId of Host is null");
588 return;
589 }
590 VirtualPortId virtualPortId = VirtualPortId.portId(ifaceId);
591 VirtualPort virtualPort = virtualPortService.getPort(virtualPortId);
592 if (virtualPort == null) {
lishuai858efd32015-12-04 14:30:36 +0800593 virtualPort = VtnData.getPort(vPortStore, virtualPortId);
lishuai6c56f5e2015-11-17 16:38:19 +0800594 }
lishuai74f2d532015-12-10 17:03:34 +0800595 Iterator<FixedIp> fixip = virtualPort.fixedIps().iterator();
596 SubnetId subnetId = null;
597 if (fixip.hasNext()) {
598 subnetId = fixip.next().subnetId();
599 }
600 if (subnetId != null) {
601 Map<HostId, Host> hosts = new ConcurrentHashMap();
602 if (hostsOfSubnet.get(subnetId) != null) {
603 hosts = hostsOfSubnet.get(subnetId);
604 }
605 if (type == Objective.Operation.ADD) {
606 hosts.put(host.id(), host);
607 hostsOfSubnet.put(subnetId, hosts);
608 } else if (type == Objective.Operation.REMOVE) {
609 hosts.remove(host.id());
610 if (hosts.size() != 0) {
611 hostsOfSubnet.put(subnetId, hosts);
612 } else {
613 hostsOfSubnet.remove(subnetId);
614 }
615 }
616 }
lishuai6c56f5e2015-11-17 16:38:19 +0800617
618 Iterable<Device> devices = deviceService.getAvailableDevices();
619 PortNumber inPort = host.location().port();
620 MacAddress mac = host.mac();
621 Device device = deviceService.getDevice(deviceId);
622 String controllerIp = VtnData.getControllerIpOfSwitch(device);
623 IpAddress ipAddress = IpAddress.valueOf(controllerIp);
624 TenantNetwork network = tenantNetworkService.getNetwork(virtualPort.networkId());
625 if (network == null) {
626 log.error("Can't find network of the host");
627 return;
628 }
629 SegmentationId segmentationId = network.segmentationId();
630 // Get all the tunnel PortNumber in the current node
631 Iterable<Port> ports = deviceService.getPorts(deviceId);
632 Collection<PortNumber> localTunnelPorts = VtnData.getLocalTunnelPorts(ports);
633 // Get all the local vm's PortNumber in the current node
634 Map<TenantNetworkId, Set<PortNumber>> localHostPorts = switchOfLocalHostPorts
jiangruic69a7fd2015-11-19 15:40:01 +0800635 .get(deviceId).getNetworkOfLocalHostPorts();
lishuai6c56f5e2015-11-17 16:38:19 +0800636 Set<PortNumber> networkOflocalHostPorts = localHostPorts.get(network.id());
jiangruic69a7fd2015-11-19 15:40:01 +0800637 for (PortNumber p : localTunnelPorts) {
638 programGroupTable(deviceId, appId, p, devices, type);
639 }
Wu wenbinacc10ea2016-05-06 16:48:56 +0800640 Subnet subnet = subnetService.getSubnet(subnetId);
641 String deviceOwner = virtualPort.deviceOwner();
642 if (deviceOwner != null) {
643 if (deviceOwner.equalsIgnoreCase("network:dhcp")) {
644 Sets.newHashSet(devices).stream()
645 .filter(d -> d.type() == Device.Type.SWITCH)
646 .forEach(d -> {
647 if (subnet != null) {
648 IpAddress dstIp = IpAddress
649 .valueOf(USERDATA_IP);
650 classifierService
651 .programUserdataClassifierRules(d.id(),
652 subnet.cidr(),
653 dstIp,
654 mac,
655 segmentationId,
656 type);
657 }
658 });
659 }
660 }
lishuai6c56f5e2015-11-17 16:38:19 +0800661 if (type == Objective.Operation.ADD) {
lishuai7dc63d92015-11-27 17:15:25 +0800662 vPortStore.put(virtualPortId, virtualPort);
lishuai6c56f5e2015-11-17 16:38:19 +0800663 if (networkOflocalHostPorts == null) {
664 networkOflocalHostPorts = new HashSet<PortNumber>();
665 localHostPorts.putIfAbsent(network.id(), networkOflocalHostPorts);
666 }
667 networkOflocalHostPorts.add(inPort);
jiangruic69a7fd2015-11-19 15:40:01 +0800668 l2ForwardService.programLocalBcastRules(deviceId, segmentationId,
669 inPort, networkOflocalHostPorts,
670 localTunnelPorts,
671 type);
lishuai6c56f5e2015-11-17 16:38:19 +0800672 classifierService.programTunnelIn(deviceId, segmentationId,
673 localTunnelPorts,
674 type);
675 } else if (type == Objective.Operation.REMOVE) {
jiangruic69a7fd2015-11-19 15:40:01 +0800676 if (networkOflocalHostPorts != null) {
677 l2ForwardService.programLocalBcastRules(deviceId, segmentationId,
678 inPort, networkOflocalHostPorts,
679 localTunnelPorts,
680 type);
681 networkOflocalHostPorts.remove(inPort);
682 if (networkOflocalHostPorts.isEmpty()) {
683 classifierService.programTunnelIn(deviceId, segmentationId,
684 localTunnelPorts,
685 type);
686 switchOfLocalHostPorts.get(deviceId).getNetworkOfLocalHostPorts()
687 .remove(virtualPort.networkId());
688 }
lishuai6c56f5e2015-11-17 16:38:19 +0800689 }
690 }
691
jiangruic69a7fd2015-11-19 15:40:01 +0800692 l2ForwardService.programLocalOut(deviceId, segmentationId, inPort, mac,
693 type);
694
lishuai6c56f5e2015-11-17 16:38:19 +0800695 l2ForwardService.programTunnelBcastRules(deviceId, segmentationId,
696 networkOflocalHostPorts,
697 localTunnelPorts,
698 type);
699
700 programTunnelOuts(devices, ipAddress, segmentationId, mac,
701 type);
702
703 classifierService.programLocalIn(deviceId, segmentationId, inPort, mac,
704 appId, type);
705 }
706
707 private void programTunnelOuts(Iterable<Device> devices,
708 IpAddress ipAddress,
709 SegmentationId segmentationId,
710 MacAddress dstMac,
711 Objective.Operation type) {
jiangruic69a7fd2015-11-19 15:40:01 +0800712 String tunnelName = "vxlan-" + DEFAULT_IP;
lishuai6c56f5e2015-11-17 16:38:19 +0800713 Sets.newHashSet(devices).stream()
jiangruic69a7fd2015-11-19 15:40:01 +0800714 .filter(d -> d.type() == Device.Type.CONTROLLER)
715 .filter(d -> !("ovsdb:" + ipAddress).equals(d.id().toString()))
716 .forEach(d -> {
lishuai6c56f5e2015-11-17 16:38:19 +0800717 DriverHandler handler = driverService.createHandler(d.id());
718 BridgeConfig bridgeConfig = handler
719 .behaviour(BridgeConfig.class);
720 Collection<BridgeDescription> bridgeDescriptions = bridgeConfig
721 .getBridges();
lishuai6c56f5e2015-11-17 16:38:19 +0800722 Iterator<BridgeDescription> it = bridgeDescriptions
723 .iterator();
lishuai70304d12015-12-14 17:15:26 +0800724 while (it.hasNext()) {
lishuai6c56f5e2015-11-17 16:38:19 +0800725 BridgeDescription sw = it.next();
lishuai70304d12015-12-14 17:15:26 +0800726 if (sw.bridgeName().name().equals(VtnConfig.DEFAULT_BRIDGE_NAME)) {
727 List<Port> ports = deviceService.getPorts(sw.deviceId());
728 ports.stream()
729 .filter(p -> p.annotations().value(AnnotationKeys.PORT_NAME)
730 .equalsIgnoreCase(tunnelName))
731 .forEach(p -> {
732 l2ForwardService.programTunnelOut(sw.deviceId(),
733 segmentationId, p.number(),
734 dstMac, type, ipAddress);
735 });
736 break;
737 }
lishuai6c56f5e2015-11-17 16:38:19 +0800738 }
739 });
740 }
741
742 private class InnerDeviceListener implements DeviceListener {
743
744 @Override
745 public void event(DeviceEvent event) {
746 Device device = event.subject();
747 if (Device.Type.CONTROLLER == device.type()) {
748 if (DeviceEvent.Type.DEVICE_ADDED == event.type()) {
749 onControllerDetected(device);
750 }
751 if (DeviceEvent.Type.DEVICE_AVAILABILITY_CHANGED == event.type()) {
752 if (deviceService.isAvailable(device.id())) {
753 onControllerDetected(device);
754 } else {
755 onControllerVanished(device);
756 }
757 }
758 } else if (Device.Type.SWITCH == device.type()) {
759 if (DeviceEvent.Type.DEVICE_ADDED == event.type()) {
760 onOvsDetected(device);
761 }
762 if (DeviceEvent.Type.DEVICE_AVAILABILITY_CHANGED == event.type()) {
763 if (deviceService.isAvailable(device.id())) {
764 onOvsDetected(device);
765 } else {
766 onOvsVanished(device);
767 }
768 }
769 } else {
770 log.info("Do nothing for this device type");
771 }
772 }
773 }
774
775 private class InnerHostListener implements HostListener {
776
777 @Override
778 public void event(HostEvent event) {
779 Host host = event.subject();
780 if (HostEvent.Type.HOST_ADDED == event.type()) {
781 onHostDetected(host);
782 } else if (HostEvent.Type.HOST_REMOVED == event.type()) {
783 onHostVanished(host);
784 } else if (HostEvent.Type.HOST_UPDATED == event.type()) {
785 onHostVanished(host);
786 onHostDetected(host);
787 }
788 }
789
790 }
791
792 // Local Host Ports of Network.
793 private class NetworkOfLocalHostPorts {
794 private final Map<TenantNetworkId, Set<PortNumber>> networkOfLocalHostPorts =
795 new HashMap<TenantNetworkId, Set<PortNumber>>();
796
797 public Map<TenantNetworkId, Set<PortNumber>> getNetworkOfLocalHostPorts() {
798 return networkOfLocalHostPorts;
799 }
800 }
801
jiangruic69a7fd2015-11-19 15:40:01 +0800802 private void programGroupTable(DeviceId deviceId, ApplicationId appid,
803 PortNumber portNumber, Iterable<Device> devices, Objective.Operation type) {
804 if (type.equals(Objective.Operation.REMOVE)) {
805 return;
806 }
807
808 List<GroupBucket> buckets = Lists.newArrayList();
809 Sets.newHashSet(devices)
810 .stream()
811 .filter(d -> d.type() == Device.Type.CONTROLLER)
812 .filter(d -> !deviceId.equals(d.id()))
813 .forEach(d -> {
814 String ipAddress = d.annotations()
815 .value(CONTROLLER_IP_KEY);
816 Ip4Address dst = Ip4Address.valueOf(ipAddress);
817 Builder builder = DefaultTrafficTreatment.builder();
818
819 DriverHandler handler = driverService.createHandler(deviceId);
820 ExtensionTreatmentResolver resolver = handler.behaviour(ExtensionTreatmentResolver.class);
821 ExtensionTreatment treatment = resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type());
822 try {
823 treatment.setPropertyValue("tunnelDst", dst);
824 } catch (Exception e) {
825 log.error("Failed to get extension instruction to set tunnel dst {}", deviceId);
826 }
827
828 builder.extension(treatment, deviceId);
829 builder.setOutput(portNumber);
830 GroupBucket bucket = DefaultGroupBucket
831 .createAllGroupBucket(builder.build());
832 buckets.add(bucket);
833 });
834 final GroupKey key = new DefaultGroupKey(APP_ID.getBytes());
835 GroupDescription groupDescription = new DefaultGroupDescription(deviceId,
836 GroupDescription.Type.ALL,
837 new GroupBuckets(buckets),
838 key,
839 L2ForwardServiceImpl.GROUP_ID,
840 appid);
841 groupService.addGroup(groupDescription);
842 }
lishuai858efd32015-12-04 14:30:36 +0800843
844 private class VtnL3EventListener implements VtnRscListener {
845 @Override
846 public void event(VtnRscEvent event) {
847 VtnRscEventFeedback l3Feedback = event.subject();
848 if (VtnRscEvent.Type.ROUTER_INTERFACE_PUT == event.type()) {
849 onRouterInterfaceDetected(l3Feedback);
850 } else
851 if (VtnRscEvent.Type.ROUTER_INTERFACE_DELETE == event.type()) {
852 onRouterInterfaceVanished(l3Feedback);
lishuai762df812016-01-08 11:51:15 +0800853 } else if (VtnRscEvent.Type.FLOATINGIP_BIND == event.type()) {
lishuai858efd32015-12-04 14:30:36 +0800854 onFloatingIpDetected(l3Feedback);
lishuai762df812016-01-08 11:51:15 +0800855 } else if (VtnRscEvent.Type.FLOATINGIP_UNBIND == event.type()) {
lishuai858efd32015-12-04 14:30:36 +0800856 onFloatingIpVanished(l3Feedback);
857 }
858 }
859
860 }
861
862 @Override
863 public void onRouterInterfaceDetected(VtnRscEventFeedback l3Feedback) {
864 Objective.Operation operation = Objective.Operation.ADD;
865 RouterInterface routerInf = l3Feedback.routerInterface();
lishuai762df812016-01-08 11:51:15 +0800866 VirtualPort gwPort = virtualPortService.getPort(routerInf.portId());
867 vPortStore.put(gwPort.portId(), gwPort);
lishuai858efd32015-12-04 14:30:36 +0800868 Iterable<RouterInterface> interfaces = routerInterfaceService
869 .getRouterInterfaces();
lishuai8798bbe2016-05-05 16:02:03 +0800870 Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces).stream()
871 .filter(r -> r.tenantId().equals(routerInf.tenantId()))
872 .filter(r -> r.routerId().equals(routerInf.routerId()))
lishuai858efd32015-12-04 14:30:36 +0800873 .collect(Collectors.toSet());
lishuai8798bbe2016-05-05 16:02:03 +0800874 TenantRouter tenantRouter = TenantRouter
875 .tenantRouter(routerInf.tenantId(), routerInf.routerId());
876 if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
lishuai858efd32015-12-04 14:30:36 +0800877 programRouterInterface(routerInf, operation);
878 } else {
879 if (interfacesSet.size() >= SUBNET_NUM) {
880 programInterfacesSet(interfacesSet, operation);
881 }
882 }
lishuaibee5eca2016-05-10 19:17:37 +0800883 // apply L3 arp flows
884 applyL3ArpFlows(null, gwPort, operation);
lishuai858efd32015-12-04 14:30:36 +0800885 }
886
887 @Override
888 public void onRouterInterfaceVanished(VtnRscEventFeedback l3Feedback) {
889 Objective.Operation operation = Objective.Operation.REMOVE;
890 RouterInterface routerInf = l3Feedback.routerInterface();
891 Iterable<RouterInterface> interfaces = routerInterfaceService
892 .getRouterInterfaces();
893 Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces)
894 .stream().filter(r -> r.tenantId().equals(routerInf.tenantId()))
895 .collect(Collectors.toSet());
lishuai8798bbe2016-05-05 16:02:03 +0800896 TenantRouter tenantRouter = TenantRouter
897 .tenantRouter(routerInf.tenantId(), routerInf.routerId());
898 if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
lishuai858efd32015-12-04 14:30:36 +0800899 programRouterInterface(routerInf, operation);
900 if (interfacesSet.size() == 1) {
lishuai8798bbe2016-05-05 16:02:03 +0800901 routerInfFlagOfTenantRouter.remove(tenantRouter);
lishuai858efd32015-12-04 14:30:36 +0800902 interfacesSet.stream().forEach(r -> {
903 programRouterInterface(r, operation);
904 });
905 }
906 }
lishuai762df812016-01-08 11:51:15 +0800907 VirtualPort gwPort = virtualPortService.getPort(routerInf.portId());
908 if (gwPort == null) {
909 gwPort = VtnData.getPort(vPortStore, routerInf.portId());
910 }
911 vPortStore.remove(gwPort.portId());
lishuaibee5eca2016-05-10 19:17:37 +0800912 // apply L3 arp flows
913 applyL3ArpFlows(null, gwPort, operation);
lishuai858efd32015-12-04 14:30:36 +0800914 }
915
916 @Override
917 public void onFloatingIpDetected(VtnRscEventFeedback l3Feedback) {
Bob zhou59a21062016-05-12 19:40:05 +0800918 floatingIpStore.put(l3Feedback.floatingIp().floatingIp(),
919 l3Feedback.floatingIp());
lishuai762df812016-01-08 11:51:15 +0800920 programFloatingIpEvent(l3Feedback, VtnRscEvent.Type.FLOATINGIP_BIND);
lishuai858efd32015-12-04 14:30:36 +0800921 }
922
923 @Override
924 public void onFloatingIpVanished(VtnRscEventFeedback l3Feedback) {
Bob zhou59a21062016-05-12 19:40:05 +0800925 floatingIpStore.remove(l3Feedback.floatingIp().floatingIp());
lishuai762df812016-01-08 11:51:15 +0800926 programFloatingIpEvent(l3Feedback, VtnRscEvent.Type.FLOATINGIP_UNBIND);
lishuai858efd32015-12-04 14:30:36 +0800927 }
928
929 private void programInterfacesSet(Set<RouterInterface> interfacesSet,
930 Objective.Operation operation) {
931 int subnetVmNum = 0;
932 for (RouterInterface r : interfacesSet) {
933 // Get all the host of the subnet
934 Map<HostId, Host> hosts = hostsOfSubnet.get(r.subnetId());
lishuaid6f0c9e2015-12-16 11:40:01 +0800935 if (hosts != null && hosts.size() > 0) {
lishuai858efd32015-12-04 14:30:36 +0800936 subnetVmNum++;
937 if (subnetVmNum >= SUBNET_NUM) {
lishuai8798bbe2016-05-05 16:02:03 +0800938 TenantRouter tenantRouter = TenantRouter
939 .tenantRouter(r.tenantId(), r.routerId());
940 routerInfFlagOfTenantRouter.put(tenantRouter, true);
lishuai858efd32015-12-04 14:30:36 +0800941 interfacesSet.stream().forEach(f -> {
942 programRouterInterface(f, operation);
943 });
944 break;
945 }
946 }
947 }
948 }
949
950 private void programRouterInterface(RouterInterface routerInf,
951 Objective.Operation operation) {
lishuai8798bbe2016-05-05 16:02:03 +0800952 TenantRouter tenantRouter = TenantRouter
953 .tenantRouter(routerInf.tenantId(), routerInf.routerId());
954 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
lishuai858efd32015-12-04 14:30:36 +0800955 // Get all the host of the subnet
956 Map<HostId, Host> hosts = hostsOfSubnet.get(routerInf.subnetId());
957 hosts.values().stream().forEach(h -> {
958 applyEastWestL3Flows(h, l3vni, operation);
959 });
960 }
961
lishuaibee5eca2016-05-10 19:17:37 +0800962 private void applyL3ArpFlows(DeviceId deviceId, VirtualPort gwPort,
963 Objective.Operation operation) {
964 IpAddress ip = null;
965 Iterator<FixedIp> gwIps = gwPort.fixedIps().iterator();
966 if (gwIps.hasNext()) {
967 ip = gwIps.next().ip();
968 }
969 IpAddress gwIp = ip;
970 MacAddress gwMac = gwPort.macAddress();
971 TenantNetwork network = tenantNetworkService
972 .getNetwork(gwPort.networkId());
973 if (deviceId != null) {
974 // Arp rules
975 DriverHandler handler = driverService.createHandler(deviceId);
976 arpService.programArpRules(handler, deviceId, gwIp,
977 network.segmentationId(), gwMac,
978 operation);
979 } else {
980 Iterable<Device> devices = deviceService.getAvailableDevices();
981 Sets.newHashSet(devices).stream()
982 .filter(d -> Device.Type.SWITCH == d.type()).forEach(d -> {
983 // Arp rules
984 DriverHandler handler = driverService.createHandler(d.id());
985 arpService.programArpRules(handler, d.id(), gwIp,
986 network.segmentationId(), gwMac,
987 operation);
988 });
989 }
990 }
991
lishuai858efd32015-12-04 14:30:36 +0800992 private void applyEastWestL3Flows(Host h, SegmentationId l3vni,
993 Objective.Operation operation) {
994 if (!mastershipService.isLocalMaster(h.location().deviceId())) {
995 log.debug("not master device:{}", h.location().deviceId());
996 return;
997 }
998 String ifaceId = h.annotations().value(IFACEID);
999 VirtualPort hPort = virtualPortService
1000 .getPort(VirtualPortId.portId(ifaceId));
1001 if (hPort == null) {
1002 hPort = VtnData.getPort(vPortStore, VirtualPortId.portId(ifaceId));
1003 }
1004 IpAddress srcIp = null;
1005 IpAddress srcGwIp = null;
1006 MacAddress srcVmGwMac = null;
1007 SubnetId srcSubnetId = null;
1008 Iterator<FixedIp> srcIps = hPort.fixedIps().iterator();
1009 if (srcIps.hasNext()) {
1010 FixedIp fixedIp = srcIps.next();
1011 srcIp = fixedIp.ip();
1012 srcSubnetId = fixedIp.subnetId();
1013 srcGwIp = subnetService.getSubnet(srcSubnetId).gatewayIp();
1014 FixedIp fixedGwIp = FixedIp.fixedIp(srcSubnetId, srcGwIp);
1015 VirtualPort gwPort = virtualPortService.getPort(fixedGwIp);
1016 if (gwPort == null) {
1017 gwPort = VtnData.getPort(vPortStore, fixedGwIp);
1018 }
1019 srcVmGwMac = gwPort.macAddress();
1020 }
1021 TenantNetwork network = tenantNetworkService
1022 .getNetwork(hPort.networkId());
lishuaibee5eca2016-05-10 19:17:37 +08001023 IpAddress dstVmIP = srcIp;
1024 MacAddress dstVmGwMac = srcVmGwMac;
1025 TenantId tenantId = hPort.tenantId();
lishuai858efd32015-12-04 14:30:36 +08001026 // Classifier rules
lishuaibee5eca2016-05-10 19:17:37 +08001027 if (operation == Objective.Operation.ADD) {
1028 sendEastWestL3Flows(h, srcVmGwMac, l3vni, srcGwIp, network,
1029 dstVmIP, dstVmGwMac, operation);
1030 } else if (operation == Objective.Operation.REMOVE) {
1031 FloatingIp floatingIp = null;
1032 Iterable<FloatingIp> floatingIps = floatingIpService.getFloatingIps();
1033 Set<FloatingIp> floatingIpSet = Sets.newHashSet(floatingIps).stream()
1034 .filter(f -> f.tenantId().equals(tenantId))
1035 .collect(Collectors.toSet());
1036 for (FloatingIp f : floatingIpSet) {
1037 IpAddress fixedIp = f.fixedIp();
1038 if (fixedIp != null && fixedIp.equals(srcIp)) {
1039 floatingIp = f;
1040 break;
1041 }
1042 }
1043 if (floatingIp == null) {
1044 sendEastWestL3Flows(h, srcVmGwMac, l3vni, srcGwIp, network,
1045 dstVmIP, dstVmGwMac, operation);
1046 }
1047 }
1048 }
1049
1050 private void sendEastWestL3Flows(Host h, MacAddress srcVmGwMac,
1051 SegmentationId l3vni, IpAddress srcGwIp,
1052 TenantNetwork network, IpAddress dstVmIP,
1053 MacAddress dstVmGwMac,
1054 Objective.Operation operation) {
lishuai858efd32015-12-04 14:30:36 +08001055 classifierService
1056 .programL3InPortClassifierRules(h.location().deviceId(),
1057 h.location().port(), h.mac(),
1058 srcVmGwMac, l3vni, operation);
lishuaibee5eca2016-05-10 19:17:37 +08001059 classifierService
1060 .programArpClassifierRules(h.location().deviceId(),
1061 h.location().port(), srcGwIp,
1062 network.segmentationId(), operation);
lishuai858efd32015-12-04 14:30:36 +08001063 Iterable<Device> devices = deviceService.getAvailableDevices();
lishuai858efd32015-12-04 14:30:36 +08001064 Sets.newHashSet(devices).stream()
1065 .filter(d -> Device.Type.SWITCH == d.type()).forEach(d -> {
1066 // L3FWD rules
lishuaibee5eca2016-05-10 19:17:37 +08001067 l3ForwardService.programRouteRules(d.id(), l3vni, dstVmIP,
lishuai858efd32015-12-04 14:30:36 +08001068 network.segmentationId(),
lishuaibee5eca2016-05-10 19:17:37 +08001069 dstVmGwMac, h.mac(),
lishuai858efd32015-12-04 14:30:36 +08001070 operation);
1071 });
1072 }
1073
1074 private void programFloatingIpEvent(VtnRscEventFeedback l3Feedback,
1075 VtnRscEvent.Type type) {
1076 FloatingIp floaingIp = l3Feedback.floatingIp();
1077 if (floaingIp != null) {
1078 VirtualPortId vmPortId = floaingIp.portId();
1079 VirtualPort vmPort = virtualPortService.getPort(vmPortId);
1080 VirtualPort fipPort = virtualPortService
1081 .getPort(floaingIp.networkId(), floaingIp.floatingIp());
1082 if (vmPort == null) {
1083 vmPort = VtnData.getPort(vPortStore, vmPortId);
1084 }
1085 if (fipPort == null) {
1086 fipPort = VtnData.getPort(vPortStore, floaingIp.networkId(),
1087 floaingIp.floatingIp());
1088 }
1089 Set<Host> hostSet = hostService.getHostsByMac(vmPort.macAddress());
1090 Host host = null;
1091 for (Host h : hostSet) {
1092 String ifaceid = h.annotations().value(IFACEID);
1093 if (ifaceid != null && ifaceid.equals(vmPortId.portId())) {
1094 host = h;
1095 break;
1096 }
1097 }
1098 if (host != null && vmPort != null && fipPort != null) {
1099 DeviceId deviceId = host.location().deviceId();
1100 Port exPort = exPortOfDevice.get(deviceId);
lishuai8798bbe2016-05-05 16:02:03 +08001101 TenantRouter tenantRouter = TenantRouter
1102 .tenantRouter(floaingIp.tenantId(), floaingIp.routerId());
1103 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
lishuai858efd32015-12-04 14:30:36 +08001104 // Floating ip BIND
lishuai762df812016-01-08 11:51:15 +08001105 if (type == VtnRscEvent.Type.FLOATINGIP_BIND) {
lishuai16c05af2016-01-21 18:39:05 +08001106 vPortStore.put(fipPort.portId(), fipPort);
lishuaibee5eca2016-05-10 19:17:37 +08001107 applyNorthSouthL3Flows(deviceId, false, tenantRouter, host,
1108 vmPort, fipPort, floaingIp, l3vni,
1109 exPort, Objective.Operation.ADD);
lishuai762df812016-01-08 11:51:15 +08001110 } else if (type == VtnRscEvent.Type.FLOATINGIP_UNBIND) {
lishuai858efd32015-12-04 14:30:36 +08001111 // Floating ip UNBIND
lishuaibee5eca2016-05-10 19:17:37 +08001112 applyNorthSouthL3Flows(deviceId, false, tenantRouter, host,
1113 vmPort, fipPort, floaingIp, l3vni,
1114 exPort,
lishuai858efd32015-12-04 14:30:36 +08001115 Objective.Operation.REMOVE);
lishuai16c05af2016-01-21 18:39:05 +08001116 vPortStore.remove(fipPort.portId());
lishuai858efd32015-12-04 14:30:36 +08001117 }
1118 }
1119 }
1120 }
1121
lishuaibee5eca2016-05-10 19:17:37 +08001122 private void sendNorthSouthL3Flows(DeviceId deviceId, FloatingIp floatingIp,
1123 IpAddress dstVmGwIp,
1124 MacAddress dstVmGwMac,
1125 SegmentationId l3vni,
1126 TenantNetwork vmNetwork,
1127 VirtualPort vmPort, Host host,
1128 Objective.Operation operation) {
1129 l3ForwardService
1130 .programRouteRules(deviceId, l3vni, floatingIp.fixedIp(),
1131 vmNetwork.segmentationId(), dstVmGwMac,
1132 vmPort.macAddress(), operation);
1133 classifierService.programL3InPortClassifierRules(deviceId,
1134 host.location().port(),
1135 host.mac(), dstVmGwMac,
1136 l3vni, operation);
1137 classifierService.programArpClassifierRules(deviceId, host.location()
1138 .port(), dstVmGwIp, vmNetwork.segmentationId(), operation);
1139 }
1140
1141 private void applyNorthSouthL3Flows(DeviceId deviceId, boolean hostFlag,
1142 TenantRouter tenantRouter, Host host,
lishuai858efd32015-12-04 14:30:36 +08001143 VirtualPort vmPort, VirtualPort fipPort,
1144 FloatingIp floatingIp,
lishuai8798bbe2016-05-05 16:02:03 +08001145 SegmentationId l3vni, Port exPort,
lishuai858efd32015-12-04 14:30:36 +08001146 Objective.Operation operation) {
1147 if (!mastershipService.isLocalMaster(deviceId)) {
1148 log.debug("not master device:{}", deviceId);
1149 return;
1150 }
1151 List gwIpMac = getGwIpAndMac(vmPort);
1152 IpAddress dstVmGwIp = (IpAddress) gwIpMac.get(0);
1153 MacAddress dstVmGwMac = (MacAddress) gwIpMac.get(1);
lishuai858efd32015-12-04 14:30:36 +08001154 TenantNetwork vmNetwork = tenantNetworkService
1155 .getNetwork(vmPort.networkId());
1156 TenantNetwork fipNetwork = tenantNetworkService
1157 .getNetwork(fipPort.networkId());
1158 // L3 downlink traffic flow
lishuaia3e32342015-12-07 10:59:17 +08001159 MacAddress exPortMac = MacAddress.valueOf(exPort.annotations()
1160 .value(AnnotationKeys.PORT_MAC));
lishuai858efd32015-12-04 14:30:36 +08001161 classifierService.programL3ExPortClassifierRules(deviceId, exPort.number(),
1162 floatingIp.floatingIp(), operation);
lishuai858efd32015-12-04 14:30:36 +08001163 dnatService.programRules(deviceId, floatingIp.floatingIp(),
Bob zhou59a21062016-05-12 19:40:05 +08001164 exPortMac, floatingIp.fixedIp(),
lishuai8798bbe2016-05-05 16:02:03 +08001165 l3vni, operation);
lishuai858efd32015-12-04 14:30:36 +08001166
Bob zhou59a21062016-05-12 19:40:05 +08001167 Subnet subnet = getSubnetOfFloatingIP(floatingIp);
1168 IpPrefix ipPrefix = subnet.cidr();
1169 snatService.programSnatSameSegmentUploadControllerRules(deviceId, l3vni,
1170 floatingIp.fixedIp(),
1171 floatingIp.floatingIp(),
1172 ipPrefix,
1173 operation);
lishuai858efd32015-12-04 14:30:36 +08001174 // L3 uplink traffic flow
lishuaibee5eca2016-05-10 19:17:37 +08001175 if (operation == Objective.Operation.ADD) {
1176 sendNorthSouthL3Flows(deviceId, floatingIp, dstVmGwIp, dstVmGwMac,
1177 l3vni, vmNetwork, vmPort, host, operation);
Bob zhou59a21062016-05-12 19:40:05 +08001178 l2ForwardService
1179 .programExternalOut(deviceId, fipNetwork.segmentationId(),
1180 exPort.number(), exPortMac, operation);
lishuaibee5eca2016-05-10 19:17:37 +08001181 } else if (operation == Objective.Operation.REMOVE) {
1182 if (hostFlag || (!hostFlag
1183 && routerInfFlagOfTenantRouter.get(tenantRouter) == null)) {
1184 sendNorthSouthL3Flows(deviceId, floatingIp, dstVmGwIp, dstVmGwMac,
1185 l3vni, vmNetwork, vmPort, host, operation);
1186 }
1187 Iterable<FloatingIp> floatingIps = floatingIpService.getFloatingIps();
1188 boolean exPortFlag = true;
1189 if (floatingIps != null) {
1190 Set<FloatingIp> floatingIpSet = Sets.newHashSet(floatingIps);
1191 for (FloatingIp fip : floatingIpSet) {
1192 if (fip.fixedIp() != null) {
1193 exPortFlag = false;
1194 break;
1195 }
1196 }
1197 }
1198 if (exPortFlag) {
Bob zhou59a21062016-05-12 19:40:05 +08001199 l2ForwardService.programExternalOut(deviceId,
1200 fipNetwork.segmentationId(),
1201 exPort.number(), exPortMac,
1202 operation);
lishuaibee5eca2016-05-10 19:17:37 +08001203 }
Bob zhou59a21062016-05-12 19:40:05 +08001204 removeRulesInSnat(deviceId, floatingIp.fixedIp());
lishuaibee5eca2016-05-10 19:17:37 +08001205 }
lishuai858efd32015-12-04 14:30:36 +08001206 }
1207
1208 private Port getExPort(DeviceId deviceId) {
1209 List<Port> ports = deviceService.getPorts(deviceId);
1210 Port exPort = null;
1211 for (Port port : ports) {
1212 String portName = port.annotations().value(AnnotationKeys.PORT_NAME);
lishuaib43dbf72016-01-06 11:11:35 +08001213 Versioned<String> exPortVersioned = exPortMap.get(EX_PORT_KEY);
1214 if (portName != null && exPortVersioned != null && portName.
1215 equals(exPortVersioned.value())) {
lishuai858efd32015-12-04 14:30:36 +08001216 exPort = port;
1217 break;
1218 }
1219 }
1220 return exPort;
1221 }
1222
1223 private List getGwIpAndMac(VirtualPort port) {
1224 List list = new ArrayList();
1225 MacAddress gwMac = null;
1226 SubnetId subnetId = null;
1227 IpAddress gwIp = null;
1228 Iterator<FixedIp> fixips = port.fixedIps().iterator();
1229 if (fixips.hasNext()) {
1230 FixedIp fixip = fixips.next();
1231 subnetId = fixip.subnetId();
1232 gwIp = subnetService.getSubnet(subnetId).gatewayIp();
1233 FixedIp fixedGwIp = FixedIp.fixedIp(fixip.subnetId(), gwIp);
1234 VirtualPort gwPort = virtualPortService.getPort(fixedGwIp);
1235 if (gwPort == null) {
1236 gwPort = VtnData.getPort(vPortStore, fixedGwIp);
1237 }
1238 gwMac = gwPort.macAddress();
1239 }
1240 list.add(gwIp);
1241 list.add(gwMac);
1242 return list;
1243 }
1244
lishuai858efd32015-12-04 14:30:36 +08001245 private void applyHostMonitoredL3Rules(Host host,
1246 Objective.Operation operation) {
1247 String ifaceId = host.annotations().value(IFACEID);
1248 DeviceId deviceId = host.location().deviceId();
1249 VirtualPortId portId = VirtualPortId.portId(ifaceId);
1250 VirtualPort port = virtualPortService.getPort(portId);
1251 if (port == null) {
1252 port = VtnData.getPort(vPortStore, portId);
1253 }
1254 TenantId tenantId = port.tenantId();
1255 Port exPort = exPortOfDevice.get(deviceId);
lishuai858efd32015-12-04 14:30:36 +08001256 Iterator<FixedIp> fixips = port.fixedIps().iterator();
1257 SubnetId sid = null;
1258 IpAddress hostIp = null;
1259 if (fixips.hasNext()) {
1260 FixedIp fixip = fixips.next();
1261 sid = fixip.subnetId();
1262 hostIp = fixip.ip();
1263 }
1264 final SubnetId subnetId = sid;
1265 // L3 internal network access to each other
1266 Iterable<RouterInterface> interfaces = routerInterfaceService
1267 .getRouterInterfaces();
lishuai8798bbe2016-05-05 16:02:03 +08001268 Set<RouterInterface> hostInterfaces = Sets.newHashSet(interfaces)
lishuai858efd32015-12-04 14:30:36 +08001269 .stream().filter(r -> r.tenantId().equals(tenantId))
lishuai8798bbe2016-05-05 16:02:03 +08001270 .filter(r -> r.subnetId().equals(subnetId))
lishuai858efd32015-12-04 14:30:36 +08001271 .collect(Collectors.toSet());
lishuai8798bbe2016-05-05 16:02:03 +08001272 hostInterfaces.stream().forEach(routerInf -> {
1273 Set<RouterInterface> interfacesSet = Sets.newHashSet(interfaces)
1274 .stream().filter(r -> r.tenantId().equals(tenantId))
1275 .filter(r -> r.routerId().equals(routerInf.routerId()))
1276 .collect(Collectors.toSet());
1277 long count = interfacesSet.stream()
1278 .filter(r -> !r.subnetId().equals(subnetId)).count();
1279 if (count > 0) {
1280 TenantRouter tenantRouter = TenantRouter
1281 .tenantRouter(routerInf.tenantId(), routerInf.routerId());
1282 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
1283 if (operation == Objective.Operation.ADD) {
1284 if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
1285 applyEastWestL3Flows(host, l3vni, operation);
1286 } else {
1287 if (interfacesSet.size() > 1) {
1288 programInterfacesSet(interfacesSet, operation);
1289 }
1290 }
1291 } else if (operation == Objective.Operation.REMOVE) {
1292 if (routerInfFlagOfTenantRouter.get(tenantRouter) != null) {
1293 applyEastWestL3Flows(host, l3vni, operation);
lishuai858efd32015-12-04 14:30:36 +08001294 }
1295 }
lishuai858efd32015-12-04 14:30:36 +08001296 }
lishuai8798bbe2016-05-05 16:02:03 +08001297 });
lishuai858efd32015-12-04 14:30:36 +08001298 // L3 external and internal network access to each other
1299 FloatingIp floatingIp = null;
1300 Iterable<FloatingIp> floatingIps = floatingIpService.getFloatingIps();
1301 Set<FloatingIp> floatingIpSet = Sets.newHashSet(floatingIps).stream()
1302 .filter(f -> f.tenantId().equals(tenantId))
1303 .collect(Collectors.toSet());
1304 for (FloatingIp f : floatingIpSet) {
1305 IpAddress fixedIp = f.fixedIp();
lishuaibee5eca2016-05-10 19:17:37 +08001306 if (fixedIp != null && fixedIp.equals(hostIp)) {
lishuai858efd32015-12-04 14:30:36 +08001307 floatingIp = f;
1308 break;
1309 }
1310 }
1311 if (floatingIp != null) {
lishuai8798bbe2016-05-05 16:02:03 +08001312 TenantRouter tenantRouter = TenantRouter
1313 .tenantRouter(floatingIp.tenantId(), floatingIp.routerId());
1314 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
lishuai858efd32015-12-04 14:30:36 +08001315 VirtualPort fipPort = virtualPortService
1316 .getPort(floatingIp.networkId(), floatingIp.floatingIp());
1317 if (fipPort == null) {
1318 fipPort = VtnData.getPort(vPortStore, floatingIp.networkId(),
1319 floatingIp.floatingIp());
1320 }
lishuaibee5eca2016-05-10 19:17:37 +08001321 applyNorthSouthL3Flows(deviceId, true, tenantRouter, host, port,
1322 fipPort, floatingIp, l3vni, exPort,
1323 operation);
lishuai858efd32015-12-04 14:30:36 +08001324 }
1325 }
lishuai74f2d532015-12-10 17:03:34 +08001326
1327 public static void setExPortName(String name) {
lishuaib43dbf72016-01-06 11:11:35 +08001328 exPortMap.put(EX_PORT_KEY, name);
lishuai74f2d532015-12-10 17:03:34 +08001329 }
Bob zhou59a21062016-05-12 19:40:05 +08001330
1331 /**
1332 * Packet processor responsible for forwarding packets along their paths.
1333 */
1334 private class VtnL3PacketProcessor implements PacketProcessor {
1335
1336 @Override
1337 public void process(PacketContext context) {
1338 InboundPacket pkt = context.inPacket();
1339 ConnectPoint connectPoint = pkt.receivedFrom();
1340 DeviceId deviceId = connectPoint.deviceId();
1341 Ethernet ethPkt = pkt.parsed();
1342 if (ethPkt == null) {
1343 return;
1344 }
1345 if (ethPkt.getEtherType() == Ethernet.TYPE_ARP) {
1346 ARP arpPacket = (ARP) ethPkt.getPayload();
1347 if ((arpPacket.getOpCode() == ARP.OP_REQUEST)) {
1348 arprequestProcess(arpPacket, deviceId);
1349 } else if (arpPacket.getOpCode() == ARP.OP_REPLY) {
1350 arpresponceProcess(arpPacket, deviceId);
1351 }
1352 } else if (ethPkt.getEtherType() == Ethernet.TYPE_IPV4) {
1353 if (ethPkt.getDestinationMAC().isMulticast()) {
1354 return;
1355 }
1356 IPv4 ip = (IPv4) ethPkt.getPayload();
1357 upStreamPacketProcessor(ip, deviceId);
1358
1359 } else {
1360 return;
1361 }
1362 }
1363
1364 private void arprequestProcess(ARP arpPacket, DeviceId deviceId) {
1365 MacAddress dstMac = MacAddress
1366 .valueOf(arpPacket.getSenderHardwareAddress());
1367 IpAddress srcIp = IpAddress.valueOf(IPv4
1368 .toIPv4Address(arpPacket.getTargetProtocolAddress()));
1369 IpAddress dstIp = IpAddress.valueOf(IPv4
1370 .toIPv4Address(arpPacket.getSenderProtocolAddress()));
1371 FloatingIp floatingIp = floatingIpStore.get(srcIp);
1372 if (floatingIp == null) {
1373 return;
1374 }
1375 DeviceId deviceIdOfFloatingIp = getDeviceIdOfFloatingIP(floatingIp);
1376 if (!deviceId.equals(deviceIdOfFloatingIp)) {
1377 return;
1378 }
1379 Port exPort = exPortOfDevice.get(deviceId);
1380 MacAddress srcMac = MacAddress.valueOf(exPort.annotations()
1381 .value(AnnotationKeys.PORT_MAC));
1382 if (!downloadSnatRules(deviceId, srcMac, srcIp, dstMac, dstIp,
1383 floatingIp)) {
1384 return;
1385 }
1386 Ethernet ethernet = buildArpResponse(dstIp, dstMac, srcIp, srcMac);
1387 if (ethernet != null) {
1388 sendPacketOut(deviceId, exPort.number(), ethernet);
1389 }
1390 }
1391
1392 private void arpresponceProcess(ARP arpPacket, DeviceId deviceId) {
1393 MacAddress srcMac = MacAddress
1394 .valueOf(arpPacket.getTargetHardwareAddress());
1395 MacAddress dstMac = MacAddress
1396 .valueOf(arpPacket.getSenderHardwareAddress());
1397 IpAddress srcIp = IpAddress.valueOf(IPv4
1398 .toIPv4Address(arpPacket.getTargetProtocolAddress()));
1399 IpAddress dstIp = IpAddress.valueOf(IPv4
1400 .toIPv4Address(arpPacket.getSenderProtocolAddress()));
1401 FloatingIp floatingIp = floatingIpStore.get(srcIp);
1402 if (floatingIp == null) {
1403 return;
1404 }
1405 DeviceId deviceIdOfFloatingIp = getDeviceIdOfFloatingIP(floatingIp);
1406 if (!deviceId.equals(deviceIdOfFloatingIp)) {
1407 return;
1408 }
1409 if (!downloadSnatRules(deviceId, srcMac, srcIp, dstMac, dstIp,
1410 floatingIp)) {
1411 return;
1412 }
1413 }
1414
1415 private void upStreamPacketProcessor(IPv4 ipPacket, DeviceId deviceId) {
1416 IpAddress srcIp = IpAddress.valueOf(ipPacket.getSourceAddress());
1417 IpAddress dstIp = IpAddress.valueOf(ipPacket.getDestinationAddress());
1418 FloatingIp floatingIp = null;
1419 Collection<FloatingIp> floatingIps = floatingIpService
1420 .getFloatingIps();
1421 Set<FloatingIp> floatingIpSet = Sets.newHashSet(floatingIps)
1422 .stream().collect(Collectors.toSet());
1423 for (FloatingIp f : floatingIpSet) {
1424 IpAddress fixIp = f.fixedIp();
1425 if (fixIp != null && fixIp.equals(srcIp)) {
1426 floatingIp = f;
1427 break;
1428 }
1429 }
1430 if (floatingIp == null) {
1431 return;
1432 }
1433 Subnet subnet = getSubnetOfFloatingIP(floatingIp);
1434 IpAddress gwIp = subnet.gatewayIp();
1435 Port exportPort = exPortOfDevice.get(deviceId);
1436 MacAddress exPortMac = MacAddress.valueOf(exportPort.annotations()
1437 .value(AnnotationKeys.PORT_MAC));
1438 IpPrefix ipPrefix = subnet.cidr();
1439 if (ipPrefix == null) {
1440 return;
1441 }
1442 int mask = ipPrefix.prefixLength();
1443 if (mask <= 0) {
1444 return;
1445 }
1446 Ethernet ethernet = null;
1447 // if the same ip segment
1448 if (IpUtil.checkSameSegment(floatingIp.floatingIp(), dstIp, mask)) {
1449 ethernet = buildArpRequest(dstIp, floatingIp.floatingIp(),
1450 exPortMac);
1451 } else {
1452 ethernet = buildArpRequest(gwIp, floatingIp.floatingIp(),
1453 exPortMac);
1454 }
1455 if (ethernet != null) {
1456 sendPacketOut(deviceId, exportPort.number(), ethernet);
1457 }
1458 }
1459 }
1460
1461 private Ethernet buildArpRequest(IpAddress targetIp, IpAddress sourceIp,
1462 MacAddress sourceMac) {
1463 ARP arp = new ARP();
1464 arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
1465 .setHardwareAddressLength((byte) Ethernet.DATALAYER_ADDRESS_LENGTH)
1466 .setProtocolType(ARP.PROTO_TYPE_IP)
1467 .setProtocolAddressLength((byte) Ip4Address.BYTE_LENGTH)
1468 .setOpCode(ARP.OP_REQUEST);
1469
1470 arp.setSenderHardwareAddress(sourceMac.toBytes())
1471 .setSenderProtocolAddress(sourceIp.getIp4Address().toInt())
1472 .setTargetHardwareAddress(ZERO_MAC_ADDRESS)
1473 .setTargetProtocolAddress(targetIp.getIp4Address().toInt());
1474
1475 Ethernet ethernet = new Ethernet();
1476 ethernet.setEtherType(Ethernet.TYPE_ARP)
1477 .setDestinationMACAddress(MacAddress.BROADCAST)
1478 .setSourceMACAddress(sourceMac)
1479 .setPayload(arp);
1480
1481 ethernet.setPad(true);
1482 return ethernet;
1483 }
1484
1485 private Ethernet buildArpResponse(IpAddress targetIp, MacAddress targetMac,
1486 IpAddress sourceIp, MacAddress sourceMac) {
1487 ARP arp = new ARP();
1488 arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
1489 .setHardwareAddressLength((byte) Ethernet.DATALAYER_ADDRESS_LENGTH)
1490 .setProtocolType(ARP.PROTO_TYPE_IP)
1491 .setProtocolAddressLength((byte) Ip4Address.BYTE_LENGTH)
1492 .setOpCode(ARP.OP_REPLY);
1493
1494 arp.setSenderHardwareAddress(sourceMac.toBytes())
1495 .setSenderProtocolAddress(sourceIp.getIp4Address().toInt())
1496 .setTargetHardwareAddress(targetMac.toBytes())
1497 .setTargetProtocolAddress(targetIp.getIp4Address().toInt());
1498
1499 Ethernet ethernet = new Ethernet();
1500 ethernet.setEtherType(Ethernet.TYPE_ARP)
1501 .setDestinationMACAddress(targetMac)
1502 .setSourceMACAddress(sourceMac)
1503 .setPayload(arp);
1504
1505 ethernet.setPad(true);
1506
1507 return ethernet;
1508 }
1509
1510 private void sendPacketOut(DeviceId deviceId, PortNumber portNumber,
1511 Ethernet payload) {
1512 TrafficTreatment treatment = DefaultTrafficTreatment.builder()
1513 .setOutput(portNumber).build();
1514 OutboundPacket packet = new DefaultOutboundPacket(deviceId, treatment,
1515 ByteBuffer
1516 .wrap(payload
1517 .serialize()));
1518 packetService.emit(packet);
1519 }
1520
1521 private Subnet getSubnetOfFloatingIP(FloatingIp floatingIp) {
1522 DeviceId exVmPortId = DeviceId
1523 .deviceId(floatingIp.id().floatingIpId().toString());
1524 Collection<VirtualPort> exVmPortList = virtualPortService
1525 .getPorts(exVmPortId);
1526 VirtualPort exVmPort = null;
1527 if (exVmPortList != null) {
1528 exVmPort = exVmPortList.iterator().next();
1529 }
1530 if (exVmPort == null) {
1531 return null;
1532 }
1533 Set<FixedIp> fixedIps = exVmPort.fixedIps();
1534 SubnetId subnetId = null;
1535 for (FixedIp f : fixedIps) {
1536 IpAddress fp = f.ip();
1537 if (fp.equals(floatingIp.floatingIp())) {
1538 subnetId = f.subnetId();
1539 break;
1540 }
1541 }
1542 if (subnetId == null) {
1543 return null;
1544 }
1545 Subnet subnet = subnetService.getSubnet(subnetId);
1546 return subnet;
1547 }
1548
1549 private DeviceId getDeviceIdOfFloatingIP(FloatingIp floatingIp) {
1550 VirtualPortId vmPortId = floatingIp.portId();
1551 VirtualPort vmPort = virtualPortService.getPort(vmPortId);
1552 if (vmPort == null) {
1553 vmPort = VtnData.getPort(vPortStore, vmPortId);
1554 }
1555 Set<Host> hostSet = hostService.getHostsByMac(vmPort.macAddress());
1556 Host host = null;
1557 for (Host h : hostSet) {
1558 String ifaceid = h.annotations().value(IFACEID);
1559 if (ifaceid != null && ifaceid.equals(vmPortId.portId())) {
1560 host = h;
1561 break;
1562 }
1563 }
1564 if (host == null) {
1565 return null;
1566 } else {
1567 return host.location().deviceId();
1568 }
1569 }
1570
1571 private boolean downloadSnatRules(DeviceId deviceId, MacAddress srcMac,
1572 IpAddress srcIp, MacAddress dstMac,
1573 IpAddress dstIp, FloatingIp floatingIp) {
1574 TenantNetwork exNetwork = tenantNetworkService
1575 .getNetwork(floatingIp.networkId());
1576 IpAddress fixedIp = floatingIp.fixedIp();
1577 VirtualPortId vmPortId = floatingIp.portId();
1578 VirtualPort vmPort = virtualPortService.getPort(vmPortId);
1579 if (vmPort == null) {
1580 vmPort = VtnData.getPort(vPortStore, vmPortId);
1581 }
1582 Subnet subnet = getSubnetOfFloatingIP(floatingIp);
1583 IpPrefix ipPrefix = subnet.cidr();
1584 IpAddress gwIp = subnet.gatewayIp();
1585 if (ipPrefix == null) {
1586 return false;
1587 }
1588 int mask = ipPrefix.prefixLength();
1589 if (mask <= 0) {
1590 return false;
1591 }
1592 TenantRouter tenantRouter = TenantRouter
1593 .tenantRouter(floatingIp.tenantId(), floatingIp.routerId());
1594 SegmentationId l3vni = vtnRscService.getL3vni(tenantRouter);
1595 // if the same ip segment
1596 if (IpUtil.checkSameSegment(srcIp, dstIp, mask)) {
1597 snatService.programSnatSameSegmentRules(deviceId, l3vni, fixedIp,
1598 dstIp, dstMac, srcMac,
1599 srcIp,
1600 exNetwork.segmentationId(),
1601 Objective.Operation.ADD);
1602 if (dstIp.equals(gwIp)) {
1603 snatService
1604 .programSnatDiffSegmentRules(deviceId, l3vni, fixedIp,
1605 dstMac, srcMac, srcIp,
1606 exNetwork.segmentationId(),
1607 Objective.Operation.ADD);
1608 }
1609 }
1610 return true;
1611 }
1612
1613 private void removeRulesInSnat(DeviceId deviceId, IpAddress fixedIp) {
1614 for (FlowEntry f : flowRuleService.getFlowEntries(deviceId)) {
1615 if (f.tableId() == SNAT_TABLE
1616 && f.priority() > SNAT_DEFAULT_RULE_PRIORITY) {
1617 String srcIp = f.selector()
1618 .getCriterion(Criterion.Type.IPV4_SRC).toString();
1619 int priority = f.priority();
1620 if (srcIp != null && srcIp.contains(fixedIp.toString())) {
1621 log.info("Match snat rules bob");
1622 TrafficSelector selector = f.selector();
1623 TrafficTreatment treatment = f.treatment();
1624 snatService.removeSnatRules(deviceId, selector, treatment,
1625 priority,
1626 Objective.Operation.REMOVE);
1627
1628 }
1629 }
1630 }
1631 }
lishuai6c56f5e2015-11-17 16:38:19 +08001632}