blob: 2bd6ca22e0522060db84d4be80040e7c8fee5750 [file] [log] [blame]
Thomas Vachuska4f1a60c2014-10-28 13:39:07 -07001/*
Brian O'Connor5ab426f2016-04-09 01:19:45 -07002 * Copyright 2014-present Open Networking Laboratory
Thomas Vachuska4f1a60c2014-10-28 13:39:07 -07003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
Brian O'Connorabafb502014-12-02 22:26:20 -080016package org.onosproject.net.flow.impl;
alshabib57044ba2014-09-16 15:58:01 -070017
Brian O'Connord12267c2015-02-17 18:17:08 -080018import com.google.common.collect.ArrayListMultimap;
19import com.google.common.collect.Iterables;
20import com.google.common.collect.Lists;
21import com.google.common.collect.Maps;
22import com.google.common.collect.Multimap;
23import com.google.common.collect.Sets;
alshabib57044ba2014-09-16 15:58:01 -070024import org.apache.felix.scr.annotations.Activate;
25import org.apache.felix.scr.annotations.Component;
26import org.apache.felix.scr.annotations.Deactivate;
Marc De Leenheerde47caa2015-04-24 11:27:44 -070027import org.apache.felix.scr.annotations.Modified;
28import org.apache.felix.scr.annotations.Property;
alshabib57044ba2014-09-16 15:58:01 -070029import org.apache.felix.scr.annotations.Reference;
30import org.apache.felix.scr.annotations.ReferenceCardinality;
31import org.apache.felix.scr.annotations.Service;
Jian Lid9b5f552016-03-11 18:15:31 -080032import org.onlab.util.Tools;
Marc De Leenheerde47caa2015-04-24 11:27:44 -070033import org.onosproject.cfg.ComponentConfigService;
Brian O'Connorabafb502014-12-02 22:26:20 -080034import org.onosproject.core.ApplicationId;
Brian O'Connor72cb19a2015-01-16 16:14:41 -080035import org.onosproject.core.CoreService;
36import org.onosproject.core.IdGenerator;
Thomas Vachuskac4ee7372016-02-02 16:10:09 -080037import org.onosproject.mastership.MastershipService;
Brian O'Connorabafb502014-12-02 22:26:20 -080038import org.onosproject.net.Device;
39import org.onosproject.net.DeviceId;
Thomas Vachuskac4ee7372016-02-02 16:10:09 -080040import org.onosproject.net.device.DeviceEvent;
41import org.onosproject.net.device.DeviceListener;
Brian O'Connorabafb502014-12-02 22:26:20 -080042import org.onosproject.net.device.DeviceService;
43import org.onosproject.net.flow.CompletedBatchOperation;
Charles M.C. Chan1229eca2015-05-18 06:27:52 +080044import org.onosproject.net.flow.DefaultFlowEntry;
Brian O'Connorabafb502014-12-02 22:26:20 -080045import org.onosproject.net.flow.FlowEntry;
46import org.onosproject.net.flow.FlowRule;
47import org.onosproject.net.flow.FlowRuleBatchEntry;
Brian O'Connorabafb502014-12-02 22:26:20 -080048import org.onosproject.net.flow.FlowRuleBatchEvent;
49import org.onosproject.net.flow.FlowRuleBatchOperation;
50import org.onosproject.net.flow.FlowRuleBatchRequest;
51import org.onosproject.net.flow.FlowRuleEvent;
52import org.onosproject.net.flow.FlowRuleListener;
Brian O'Connor72cb19a2015-01-16 16:14:41 -080053import org.onosproject.net.flow.FlowRuleOperation;
54import org.onosproject.net.flow.FlowRuleOperations;
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -080055import org.onosproject.net.flow.FlowRuleProgrammable;
Brian O'Connorabafb502014-12-02 22:26:20 -080056import org.onosproject.net.flow.FlowRuleProvider;
57import org.onosproject.net.flow.FlowRuleProviderRegistry;
58import org.onosproject.net.flow.FlowRuleProviderService;
59import org.onosproject.net.flow.FlowRuleService;
60import org.onosproject.net.flow.FlowRuleStore;
61import org.onosproject.net.flow.FlowRuleStoreDelegate;
Srikanth Vavilapalli95810f52015-09-14 15:49:56 -070062import org.onosproject.net.flow.TableStatisticsEntry;
Thomas Vachuskac4ee7372016-02-02 16:10:09 -080063import org.onosproject.net.provider.AbstractListenerProviderRegistry;
Brian O'Connorabafb502014-12-02 22:26:20 -080064import org.onosproject.net.provider.AbstractProviderService;
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -080065import org.onosproject.net.provider.ProviderId;
Marc De Leenheerde47caa2015-04-24 11:27:44 -070066import org.osgi.service.component.ComponentContext;
alshabib57044ba2014-09-16 15:58:01 -070067import org.slf4j.Logger;
68
Brian O'Connord12267c2015-02-17 18:17:08 -080069import java.util.Collections;
Marc De Leenheerde47caa2015-04-24 11:27:44 -070070import java.util.Dictionary;
Sho SHIMIZUb837cb72016-08-30 14:44:50 -070071import java.util.HashSet;
Brian O'Connord12267c2015-02-17 18:17:08 -080072import java.util.List;
73import java.util.Map;
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -080074import java.util.Optional;
Brian O'Connord12267c2015-02-17 18:17:08 -080075import java.util.Set;
76import java.util.concurrent.ConcurrentHashMap;
77import java.util.concurrent.ExecutorService;
78import java.util.concurrent.Executors;
Brian O'Connor72cb19a2015-01-16 16:14:41 -080079
Thomas Vachuska9b2da212014-11-10 19:30:25 -080080import static com.google.common.base.Preconditions.checkNotNull;
Charles Chan0c7c43b2016-01-14 17:39:20 -080081import static com.google.common.base.Strings.isNullOrEmpty;
Thomas Vachuskac4ee7372016-02-02 16:10:09 -080082import static org.onlab.util.Tools.get;
Brian O'Connord12267c2015-02-17 18:17:08 -080083import static org.onlab.util.Tools.groupedThreads;
Thomas Vachuska42e8cce2015-07-29 19:25:18 -070084import static org.onosproject.net.flow.FlowRuleEvent.Type.RULE_ADD_REQUESTED;
85import static org.onosproject.net.flow.FlowRuleEvent.Type.RULE_REMOVE_REQUESTED;
Changhoon Yoon541ef712015-05-23 17:18:34 +090086import static org.onosproject.security.AppGuard.checkPermission;
Thomas Vachuskac4ee7372016-02-02 16:10:09 -080087import static org.onosproject.security.AppPermission.Type.FLOWRULE_READ;
88import static org.onosproject.security.AppPermission.Type.FLOWRULE_WRITE;
Thomas Vachuska42e8cce2015-07-29 19:25:18 -070089import static org.slf4j.LoggerFactory.getLogger;
Changhoon Yoonb856b812015-08-10 03:47:19 +090090
Changhoon Yoon541ef712015-05-23 17:18:34 +090091
alshabiba7f7ca82014-09-22 11:41:23 -070092
tome4729872014-09-23 00:37:37 -070093/**
94 * Provides implementation of the flow NB & SB APIs.
95 */
Sho SHIMIZU5c396e32016-08-12 15:19:12 -070096@Component(immediate = true)
alshabib57044ba2014-09-16 15:58:01 -070097@Service
tom202175a2014-09-19 19:00:11 -070098public class FlowRuleManager
Thomas Vachuska42e8cce2015-07-29 19:25:18 -070099 extends AbstractListenerProviderRegistry<FlowRuleEvent, FlowRuleListener,
100 FlowRuleProvider, FlowRuleProviderService>
tom9b4030d2014-10-06 10:39:03 -0700101 implements FlowRuleService, FlowRuleProviderRegistry {
alshabib57044ba2014-09-16 15:58:01 -0700102
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800103 private final Logger log = getLogger(getClass());
104
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700105 public static final String FLOW_RULE_NULL = "FlowRule cannot be null";
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700106 private static final boolean ALLOW_EXTRANEOUS_RULES = false;
107
108 @Property(name = "allowExtraneousRules", boolValue = ALLOW_EXTRANEOUS_RULES,
109 label = "Allow flow rules in switch not installed by ONOS")
110 private boolean allowExtraneousRules = ALLOW_EXTRANEOUS_RULES;
111
Charles Chan0c7c43b2016-01-14 17:39:20 -0800112 @Property(name = "purgeOnDisconnection", boolValue = false,
113 label = "Purge entries associated with a device when the device goes offline")
114 private boolean purgeOnDisconnection = false;
115
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800116 private static final int DEFAULT_POLL_FREQUENCY = 30;
117 @Property(name = "fallbackFlowPollFrequency", intValue = DEFAULT_POLL_FREQUENCY,
118 label = "Frequency (in seconds) for polling flow statistics via fallback provider")
119 private int fallbackFlowPollFrequency = DEFAULT_POLL_FREQUENCY;
alshabib57044ba2014-09-16 15:58:01 -0700120
alshabibbb42cad2014-09-25 11:43:05 -0700121 private final FlowRuleStoreDelegate delegate = new InternalStoreDelegate();
Charles Chan0c7c43b2016-01-14 17:39:20 -0800122 private final DeviceListener deviceListener = new InternalDeviceListener();
tomc78acee2014-09-24 15:16:55 -0700123
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -0800124 private final FlowRuleDriverProvider driverProvider = new FlowRuleDriverProvider();
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800125
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800126 protected ExecutorService deviceInstallers =
HIGUCHI Yuta060da9a2016-03-11 19:16:35 -0800127 Executors.newFixedThreadPool(32, groupedThreads("onos/flowservice", "device-installer-%d", log));
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800128
129 protected ExecutorService operationsService =
HIGUCHI Yutad9e01052016-04-14 09:31:42 -0700130 Executors.newFixedThreadPool(32, groupedThreads("onos/flowservice", "operations-%d", log));
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800131
132 private IdGenerator idGenerator;
133
Sho SHIMIZU9f950742016-09-01 09:55:38 -0700134 private final Map<Long, FlowOperationsProcessor> pendingFlowOperations = new ConcurrentHashMap<>();
Yuta HIGUCHI9def0472014-10-23 15:51:10 -0700135
tombe988312014-09-19 18:38:47 -0700136 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
137 protected FlowRuleStore store;
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700138
alshabib57044ba2014-09-16 15:58:01 -0700139 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
Ayaka Koshibeb55524f2014-09-18 09:59:24 -0700140 protected DeviceService deviceService;
alshabib57044ba2014-09-16 15:58:01 -0700141
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800142 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
143 protected CoreService coreService;
144
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700145 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800146 protected MastershipService mastershipService;
147
148 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700149 protected ComponentConfigService cfgService;
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800150
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700151 @Activate
152 public void activate(ComponentContext context) {
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800153 modified(context);
tomc78acee2014-09-24 15:16:55 -0700154 store.setDelegate(delegate);
alshabib57044ba2014-09-16 15:58:01 -0700155 eventDispatcher.addSink(FlowRuleEvent.class, listenerRegistry);
Charles Chan0c7c43b2016-01-14 17:39:20 -0800156 deviceService.addListener(deviceListener);
157 cfgService.registerProperties(getClass());
158 idGenerator = coreService.getIdGenerator(FLOW_OP_TOPIC);
alshabib57044ba2014-09-16 15:58:01 -0700159 log.info("Started");
160 }
161
162 @Deactivate
163 public void deactivate() {
Andrea Campanella3f1c61e2016-04-01 17:30:12 -0700164 deviceService.removeListener(deviceListener);
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700165 cfgService.unregisterProperties(getClass(), false);
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800166 deviceInstallers.shutdownNow();
167 operationsService.shutdownNow();
tomc78acee2014-09-24 15:16:55 -0700168 store.unsetDelegate(delegate);
alshabib57044ba2014-09-16 15:58:01 -0700169 eventDispatcher.removeSink(FlowRuleEvent.class);
170 log.info("Stopped");
171 }
172
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700173 @Modified
174 public void modified(ComponentContext context) {
Charles Chan0c7c43b2016-01-14 17:39:20 -0800175 if (context != null) {
176 readComponentConfiguration(context);
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700177 }
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -0800178 driverProvider.init(new InternalFlowRuleProviderService(driverProvider),
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800179 deviceService, mastershipService, fallbackFlowPollFrequency);
180 }
181
182 @Override
183 protected FlowRuleProvider defaultProvider() {
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -0800184 return driverProvider;
Charles Chan0c7c43b2016-01-14 17:39:20 -0800185 }
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700186
Charles Chan0c7c43b2016-01-14 17:39:20 -0800187 /**
188 * Extracts properties from the component configuration context.
189 *
190 * @param context the component context
191 */
192 private void readComponentConfiguration(ComponentContext context) {
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700193 Dictionary<?, ?> properties = context.getProperties();
Charles Chan0c7c43b2016-01-14 17:39:20 -0800194 Boolean flag;
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700195
Jian Lid9b5f552016-03-11 18:15:31 -0800196 flag = Tools.isPropertyEnabled(properties, "allowExtraneousRules");
Charles Chan0c7c43b2016-01-14 17:39:20 -0800197 if (flag == null) {
198 log.info("AllowExtraneousRules is not configured, " +
199 "using current value of {}", allowExtraneousRules);
200 } else {
201 allowExtraneousRules = flag;
202 log.info("Configured. AllowExtraneousRules is {}",
203 allowExtraneousRules ? "enabled" : "disabled");
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700204 }
Charles Chan0c7c43b2016-01-14 17:39:20 -0800205
Jian Lid9b5f552016-03-11 18:15:31 -0800206 flag = Tools.isPropertyEnabled(properties, "purgeOnDisconnection");
Charles Chan0c7c43b2016-01-14 17:39:20 -0800207 if (flag == null) {
208 log.info("PurgeOnDisconnection is not configured, " +
209 "using current value of {}", purgeOnDisconnection);
210 } else {
211 purgeOnDisconnection = flag;
212 log.info("Configured. PurgeOnDisconnection is {}",
213 purgeOnDisconnection ? "enabled" : "disabled");
214 }
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800215
216 String s = get(properties, "fallbackFlowPollFrequency");
217 try {
218 fallbackFlowPollFrequency = isNullOrEmpty(s) ? DEFAULT_POLL_FREQUENCY : Integer.parseInt(s);
219 } catch (NumberFormatException e) {
220 fallbackFlowPollFrequency = DEFAULT_POLL_FREQUENCY;
221 }
Charles Chan0c7c43b2016-01-14 17:39:20 -0800222 }
223
alshabib57044ba2014-09-16 15:58:01 -0700224 @Override
tom9b4030d2014-10-06 10:39:03 -0700225 public int getFlowRuleCount() {
Changhoon Yoonb856b812015-08-10 03:47:19 +0900226 checkPermission(FLOWRULE_READ);
tom9b4030d2014-10-06 10:39:03 -0700227 return store.getFlowRuleCount();
228 }
229
230 @Override
alshabib1c319ff2014-10-04 20:29:09 -0700231 public Iterable<FlowEntry> getFlowEntries(DeviceId deviceId) {
Changhoon Yoonb856b812015-08-10 03:47:19 +0900232 checkPermission(FLOWRULE_READ);
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700233 return store.getFlowEntries(deviceId);
alshabib57044ba2014-09-16 15:58:01 -0700234 }
235
236 @Override
alshabib219ebaa2014-09-22 15:41:24 -0700237 public void applyFlowRules(FlowRule... flowRules) {
Changhoon Yoonb856b812015-08-10 03:47:19 +0900238 checkPermission(FLOWRULE_WRITE);
Changhoon Yoon541ef712015-05-23 17:18:34 +0900239
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800240 FlowRuleOperations.Builder builder = FlowRuleOperations.builder();
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800241 for (FlowRule flowRule : flowRules) {
242 builder.add(flowRule);
Yuta HIGUCHIf3d51bd2014-10-21 01:05:33 -0700243 }
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800244 apply(builder.build());
alshabib57044ba2014-09-16 15:58:01 -0700245 }
246
247 @Override
Kavitha Alagesanc69c66a2016-06-15 14:26:04 +0530248 public void purgeFlowRules(DeviceId deviceId) {
249 checkPermission(FLOWRULE_WRITE);
250 store.purgeFlowRule(deviceId);
251 }
252
253 @Override
alshabib57044ba2014-09-16 15:58:01 -0700254 public void removeFlowRules(FlowRule... flowRules) {
Changhoon Yoonb856b812015-08-10 03:47:19 +0900255 checkPermission(FLOWRULE_WRITE);
Changhoon Yoon541ef712015-05-23 17:18:34 +0900256
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800257 FlowRuleOperations.Builder builder = FlowRuleOperations.builder();
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800258 for (FlowRule flowRule : flowRules) {
259 builder.remove(flowRule);
Yuta HIGUCHIf3d51bd2014-10-21 01:05:33 -0700260 }
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800261 apply(builder.build());
alshabiba68eb962014-09-24 20:34:13 -0700262 }
alshabib57044ba2014-09-16 15:58:01 -0700263
alshabiba68eb962014-09-24 20:34:13 -0700264 @Override
265 public void removeFlowRulesById(ApplicationId id) {
Changhoon Yoonb856b812015-08-10 03:47:19 +0900266 checkPermission(FLOWRULE_WRITE);
Madan Jampani6a456162014-10-24 11:36:17 -0700267 removeFlowRules(Iterables.toArray(getFlowRulesById(id), FlowRule.class));
alshabiba68eb962014-09-24 20:34:13 -0700268 }
269
Bharath Thiruveedula99849dc2016-11-17 22:04:38 +0530270 @Deprecated
alshabiba68eb962014-09-24 20:34:13 -0700271 @Override
272 public Iterable<FlowRule> getFlowRulesById(ApplicationId id) {
Changhoon Yoonb856b812015-08-10 03:47:19 +0900273 checkPermission(FLOWRULE_READ);
Changhoon Yoon541ef712015-05-23 17:18:34 +0900274
Madan Jampani6a456162014-10-24 11:36:17 -0700275 Set<FlowRule> flowEntries = Sets.newHashSet();
276 for (Device d : deviceService.getDevices()) {
277 for (FlowEntry flowEntry : store.getFlowEntries(d.id())) {
278 if (flowEntry.appId() == id.id()) {
279 flowEntries.add(flowEntry);
280 }
281 }
282 }
283 return flowEntries;
alshabib57044ba2014-09-16 15:58:01 -0700284 }
285
286 @Override
Bharath Thiruveedula99849dc2016-11-17 22:04:38 +0530287 public Iterable<FlowEntry> getFlowEntriesById(ApplicationId id) {
288 checkPermission(FLOWRULE_READ);
289
290 Set<FlowEntry> flowEntries = Sets.newHashSet();
291 for (Device d : deviceService.getDevices()) {
292 for (FlowEntry flowEntry : store.getFlowEntries(d.id())) {
293 if (flowEntry.appId() == id.id()) {
294 flowEntries.add(flowEntry);
295 }
296 }
297 }
298 return flowEntries;
299 }
300
301 @Override
alshabibaa7e7de2014-11-12 19:20:44 -0800302 public Iterable<FlowRule> getFlowRulesByGroupId(ApplicationId appId, short groupId) {
Changhoon Yoonb856b812015-08-10 03:47:19 +0900303 checkPermission(FLOWRULE_READ);
Changhoon Yoon541ef712015-05-23 17:18:34 +0900304
alshabibaa7e7de2014-11-12 19:20:44 -0800305 Set<FlowRule> matches = Sets.newHashSet();
306 long toLookUp = ((long) appId.id() << 16) | groupId;
307 for (Device d : deviceService.getDevices()) {
308 for (FlowEntry flowEntry : store.getFlowEntries(d.id())) {
309 if ((flowEntry.id().value() >>> 32) == toLookUp) {
310 matches.add(flowEntry);
311 }
312 }
313 }
314 return matches;
315 }
316
317 @Override
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800318 public void apply(FlowRuleOperations ops) {
Changhoon Yoonb856b812015-08-10 03:47:19 +0900319 checkPermission(FLOWRULE_WRITE);
HIGUCHI Yutad9e01052016-04-14 09:31:42 -0700320 operationsService.execute(new FlowOperationsProcessor(ops));
alshabib902d41b2014-10-07 16:52:05 -0700321 }
322
323 @Override
alshabib57044ba2014-09-16 15:58:01 -0700324 protected FlowRuleProviderService createProviderService(
325 FlowRuleProvider provider) {
326 return new InternalFlowRuleProviderService(provider);
327 }
328
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -0800329 @Override
330 protected synchronized FlowRuleProvider getProvider(ProviderId pid) {
331 log.warn("should not be calling getProvider(ProviderId)");
332 return super.getProvider(pid);
333 }
334
335 /**
336 * {@inheritDoc}
337 * if the Device does not support {@link FlowRuleProgrammable}.
338 */
339 @Override
340 protected synchronized FlowRuleProvider getProvider(DeviceId deviceId) {
341 // if device supports FlowRuleProgrammable,
342 // use FlowRuleProgrammable via FlowRuleDriverProvider
343 return Optional.ofNullable(deviceService.getDevice(deviceId))
344 .filter(dev -> dev.is(FlowRuleProgrammable.class))
345 .<FlowRuleProvider>map(x -> driverProvider)
346 .orElseGet(() -> super.getProvider(deviceId));
347 }
348
alshabib57044ba2014-09-16 15:58:01 -0700349 private class InternalFlowRuleProviderService
tom9b4030d2014-10-06 10:39:03 -0700350 extends AbstractProviderService<FlowRuleProvider>
351 implements FlowRuleProviderService {
alshabib57044ba2014-09-16 15:58:01 -0700352
Murat Parlakisikc6759e82016-06-29 03:22:22 -0700353 final Map<FlowEntry, Long> firstSeen = Maps.newConcurrentMap();
Yuta HIGUCHIf6f50a62014-10-19 15:58:49 -0700354 final Map<FlowEntry, Long> lastSeen = Maps.newConcurrentMap();
355
Murat Parlakisikc6759e82016-06-29 03:22:22 -0700356
alshabib57044ba2014-09-16 15:58:01 -0700357 protected InternalFlowRuleProviderService(FlowRuleProvider provider) {
358 super(provider);
359 }
360
361 @Override
alshabib1c319ff2014-10-04 20:29:09 -0700362 public void flowRemoved(FlowEntry flowEntry) {
363 checkNotNull(flowEntry, FLOW_RULE_NULL);
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700364 checkValidity();
Yuta HIGUCHIf6f50a62014-10-19 15:58:49 -0700365 lastSeen.remove(flowEntry);
Murat Parlakisikc6759e82016-06-29 03:22:22 -0700366 firstSeen.remove(flowEntry);
alshabib1c319ff2014-10-04 20:29:09 -0700367 FlowEntry stored = store.getFlowEntry(flowEntry);
alshabiba68eb962014-09-24 20:34:13 -0700368 if (stored == null) {
Yuta HIGUCHI82e53262014-11-27 10:28:51 -0800369 log.debug("Rule already evicted from store: {}", flowEntry);
alshabiba68eb962014-09-24 20:34:13 -0700370 return;
371 }
Murat Parlakisikc6759e82016-06-29 03:22:22 -0700372 if (flowEntry.reason() == FlowEntry.FlowRemoveReason.HARD_TIMEOUT) {
373 ((DefaultFlowEntry) stored).setState(FlowEntry.FlowEntryState.REMOVED);
374 }
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -0800375 FlowRuleProvider frp = getProvider(flowEntry.deviceId());
alshabiba68eb962014-09-24 20:34:13 -0700376 FlowRuleEvent event = null;
377 switch (stored.state()) {
tom9b4030d2014-10-06 10:39:03 -0700378 case ADDED:
379 case PENDING_ADD:
alshabib6eb438a2014-10-01 16:39:37 -0700380 frp.applyFlowRule(stored);
tom9b4030d2014-10-06 10:39:03 -0700381 break;
382 case PENDING_REMOVE:
383 case REMOVED:
384 event = store.removeFlowRule(stored);
385 break;
386 default:
387 break;
alshabib57044ba2014-09-16 15:58:01 -0700388
alshabiba68eb962014-09-24 20:34:13 -0700389 }
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700390 if (event != null) {
alshabib1c319ff2014-10-04 20:29:09 -0700391 log.debug("Flow {} removed", flowEntry);
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700392 post(event);
393 }
alshabib57044ba2014-09-16 15:58:01 -0700394 }
395
alshabibba5ac482014-10-02 17:15:20 -0700396
alshabib1c319ff2014-10-04 20:29:09 -0700397 private void flowMissing(FlowEntry flowRule) {
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700398 checkNotNull(flowRule, FLOW_RULE_NULL);
399 checkValidity();
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -0800400 FlowRuleProvider frp = getProvider(flowRule.deviceId());
alshabibbb42cad2014-09-25 11:43:05 -0700401 FlowRuleEvent event = null;
alshabiba68eb962014-09-24 20:34:13 -0700402 switch (flowRule.state()) {
tom9b4030d2014-10-06 10:39:03 -0700403 case PENDING_REMOVE:
404 case REMOVED:
405 event = store.removeFlowRule(flowRule);
tom9b4030d2014-10-06 10:39:03 -0700406 break;
407 case ADDED:
408 case PENDING_ADD:
Charles Chan93fa7272016-01-26 22:27:02 -0800409 event = store.pendingFlowRule(flowRule);
Charles M.C. Chan1229eca2015-05-18 06:27:52 +0800410 try {
411 frp.applyFlowRule(flowRule);
412 } catch (UnsupportedOperationException e) {
413 log.warn(e.getMessage());
414 if (flowRule instanceof DefaultFlowEntry) {
Brian O'Connora3e5cd52015-12-05 15:59:19 -0800415 //FIXME modification of "stored" flow entry outside of store
Charles M.C. Chan1229eca2015-05-18 06:27:52 +0800416 ((DefaultFlowEntry) flowRule).setState(FlowEntry.FlowEntryState.FAILED);
417 }
418 }
tom9b4030d2014-10-06 10:39:03 -0700419 break;
420 default:
421 log.debug("Flow {} has not been installed.", flowRule);
alshabiba68eb962014-09-24 20:34:13 -0700422 }
423
alshabibbb42cad2014-09-25 11:43:05 -0700424 if (event != null) {
425 log.debug("Flow {} removed", flowRule);
426 post(event);
427 }
alshabib57044ba2014-09-16 15:58:01 -0700428 }
429
alshabibba5ac482014-10-02 17:15:20 -0700430 private void extraneousFlow(FlowRule flowRule) {
alshabib219ebaa2014-09-22 15:41:24 -0700431 checkNotNull(flowRule, FLOW_RULE_NULL);
432 checkValidity();
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -0800433 // getProvider is customized to favor driverProvider
alshabib2374fc92014-10-22 11:03:23 -0700434 FlowRuleProvider frp = getProvider(flowRule.deviceId());
435 frp.removeFlowRule(flowRule);
alshabib54ce5892014-09-23 17:50:51 -0700436 log.debug("Flow {} is on switch but not in store.", flowRule);
alshabib219ebaa2014-09-22 15:41:24 -0700437 }
438
alshabib1c319ff2014-10-04 20:29:09 -0700439 private void flowAdded(FlowEntry flowEntry) {
440 checkNotNull(flowEntry, FLOW_RULE_NULL);
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700441 checkValidity();
alshabib57044ba2014-09-16 15:58:01 -0700442
alshabib1c319ff2014-10-04 20:29:09 -0700443 if (checkRuleLiveness(flowEntry, store.getFlowEntry(flowEntry))) {
alshabib1c319ff2014-10-04 20:29:09 -0700444 FlowRuleEvent event = store.addOrUpdateFlowRule(flowEntry);
alshabibba5ac482014-10-02 17:15:20 -0700445 if (event == null) {
446 log.debug("No flow store event generated.");
447 } else {
Jonathan Hart58682dd2014-11-24 20:11:16 -0800448 log.trace("Flow {} {}", flowEntry, event.type());
alshabibba5ac482014-10-02 17:15:20 -0700449 post(event);
450 }
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700451 } else {
Thomas Vachuska4830d392014-11-09 17:09:56 -0800452 log.debug("Removing flow rules....");
alshabib1c319ff2014-10-04 20:29:09 -0700453 removeFlowRules(flowEntry);
Ayaka Koshibe08eabaa2014-09-17 14:59:25 -0700454 }
alshabib57044ba2014-09-16 15:58:01 -0700455 }
456
alshabib1c319ff2014-10-04 20:29:09 -0700457 private boolean checkRuleLiveness(FlowEntry swRule, FlowEntry storedRule) {
458 if (storedRule == null) {
459 return false;
460 }
Jonathan Hartbc4a7932014-10-21 11:46:00 -0700461 if (storedRule.isPermanent()) {
462 return true;
463 }
464
Yuta HIGUCHIf6f50a62014-10-19 15:58:49 -0700465 final long timeout = storedRule.timeout() * 1000;
466 final long currentTime = System.currentTimeMillis();
Murat Parlakisikc6759e82016-06-29 03:22:22 -0700467
468 // Checking flow with hardTimeout
469 if (storedRule.hardTimeout() != 0) {
470 if (!firstSeen.containsKey(storedRule)) {
471 // First time rule adding
472 firstSeen.put(storedRule, currentTime);
473 } else {
474 Long first = firstSeen.get(storedRule);
475 final long hardTimeout = storedRule.hardTimeout() * 1000;
476 if ((currentTime - first) > hardTimeout) {
477 return false;
478 }
479 }
480 }
481
alshabib85c41972014-10-03 13:48:39 -0700482 if (storedRule.packets() != swRule.packets()) {
Yuta HIGUCHIf6f50a62014-10-19 15:58:49 -0700483 lastSeen.put(storedRule, currentTime);
alshabib85c41972014-10-03 13:48:39 -0700484 return true;
485 }
Yuta HIGUCHIf6f50a62014-10-19 15:58:49 -0700486 if (!lastSeen.containsKey(storedRule)) {
487 // checking for the first time
488 lastSeen.put(storedRule, storedRule.lastSeen());
489 // Use following if lastSeen attr. was removed.
490 //lastSeen.put(storedRule, currentTime);
491 }
492 Long last = lastSeen.get(storedRule);
alshabib85c41972014-10-03 13:48:39 -0700493
Thomas Vachuskac4ee7372016-02-02 16:10:09 -0800494 // concurrently removed? let the liveness check fail
495 return last != null && (currentTime - last) <= timeout;
alshabibba5ac482014-10-02 17:15:20 -0700496 }
497
alshabib5c370ff2014-09-18 10:12:14 -0700498 @Override
alshabib1c319ff2014-10-04 20:29:09 -0700499 public void pushFlowMetrics(DeviceId deviceId, Iterable<FlowEntry> flowEntries) {
ssyoon9030fbcd92015-08-17 10:42:07 +0900500 pushFlowMetricsInternal(deviceId, flowEntries, true);
501 }
502
503 @Override
504 public void pushFlowMetricsWithoutFlowMissing(DeviceId deviceId, Iterable<FlowEntry> flowEntries) {
505 pushFlowMetricsInternal(deviceId, flowEntries, false);
506 }
507
508 private void pushFlowMetricsInternal(DeviceId deviceId, Iterable<FlowEntry> flowEntries,
509 boolean useMissingFlow) {
Jonathan Hartf44e42c2015-08-04 09:58:46 -0700510 Map<FlowEntry, FlowEntry> storedRules = Maps.newHashMap();
511 store.getFlowEntries(deviceId).forEach(f -> storedRules.put(f, f));
512
Saurav Dasfa2fa932015-03-03 11:29:48 -0800513 for (FlowEntry rule : flowEntries) {
514 try {
Jonathan Hartf44e42c2015-08-04 09:58:46 -0700515 FlowEntry storedRule = storedRules.remove(rule);
516 if (storedRule != null) {
517 if (storedRule.exactMatch(rule)) {
518 // we both have the rule, let's update some info then.
519 flowAdded(rule);
520 } else {
521 // the two rules are not an exact match - remove the
522 // switch's rule and install our rule
523 extraneousFlow(rule);
524 flowMissing(storedRule);
525 }
Saurav Dasfa2fa932015-03-03 11:29:48 -0800526 } else {
527 // the device has a rule the store does not have
Marc De Leenheerde47caa2015-04-24 11:27:44 -0700528 if (!allowExtraneousRules) {
529 extraneousFlow(rule);
530 }
alshabib93cb57f2015-02-12 17:43:26 -0800531 }
Sho SHIMIZU24a00d92015-05-05 11:11:13 -0700532 } catch (Exception e) {
Saurav Dasfa2fa932015-03-03 11:29:48 -0800533 log.debug("Can't process added or extra rule {}", e.getMessage());
alshabib93cb57f2015-02-12 17:43:26 -0800534 }
Saurav Dasfa2fa932015-03-03 11:29:48 -0800535 }
ssyoon9030fbcd92015-08-17 10:42:07 +0900536
537 // DO NOT reinstall
538 if (useMissingFlow) {
539 for (FlowEntry rule : storedRules.keySet()) {
540 try {
541 // there are rules in the store that aren't on the switch
542 log.debug("Adding rule in store, but not on switch {}", rule);
543 flowMissing(rule);
544 } catch (Exception e) {
Jonathan Hart26a8d952015-12-02 15:16:35 -0800545 log.debug("Can't add missing flow rule:", e);
ssyoon9030fbcd92015-08-17 10:42:07 +0900546 }
alshabib93cb57f2015-02-12 17:43:26 -0800547 }
Saurav Dasfa2fa932015-03-03 11:29:48 -0800548 }
alshabib5c370ff2014-09-18 10:12:14 -0700549 }
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800550
551 @Override
552 public void batchOperationCompleted(long batchId, CompletedBatchOperation operation) {
553 store.batchOperationComplete(FlowRuleBatchEvent.completed(
554 new FlowRuleBatchRequest(batchId, Collections.emptySet()),
555 operation
556 ));
557 }
Srikanth Vavilapalli95810f52015-09-14 15:49:56 -0700558
559 @Override
560 public void pushTableStatistics(DeviceId deviceId,
561 List<TableStatisticsEntry> tableStats) {
562 store.updateTableStatistics(deviceId, tableStats);
563 }
alshabib57044ba2014-09-16 15:58:01 -0700564 }
565
tomc78acee2014-09-24 15:16:55 -0700566 // Store delegate to re-post events emitted from the store.
567 private class InternalStoreDelegate implements FlowRuleStoreDelegate {
Thomas Vachuska9b2da212014-11-10 19:30:25 -0800568
Thomas Vachuska9b2da212014-11-10 19:30:25 -0800569
Madan Jampani117aaae2014-10-23 10:04:05 -0700570 // TODO: Right now we only dispatch events at individual flowEntry level.
571 // It may be more efficient for also dispatch events as a batch.
tomc78acee2014-09-24 15:16:55 -0700572 @Override
Madan Jampani117aaae2014-10-23 10:04:05 -0700573 public void notify(FlowRuleBatchEvent event) {
574 final FlowRuleBatchRequest request = event.subject();
Yuta HIGUCHIf3d51bd2014-10-21 01:05:33 -0700575 switch (event.type()) {
Madan Jampani117aaae2014-10-23 10:04:05 -0700576 case BATCH_OPERATION_REQUESTED:
Yuta HIGUCHI2fcb40c2014-11-03 14:39:10 -0800577 // Request has been forwarded to MASTER Node, and was
Sho SHIMIZUa09e1bb2016-08-01 14:25:25 -0700578 request.ops().forEach(
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800579 op -> {
Ray Milkeyf7329c72015-02-17 11:37:01 -0800580 switch (op.operator()) {
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800581 case ADD:
Sho SHIMIZU8efc8962016-08-31 15:17:44 -0700582 post(new FlowRuleEvent(RULE_ADD_REQUESTED, op.target()));
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800583 break;
584 case REMOVE:
Sho SHIMIZU8efc8962016-08-31 15:17:44 -0700585 post(new FlowRuleEvent(RULE_REMOVE_REQUESTED, op.target()));
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800586 break;
587 case MODIFY:
588 //TODO: do something here when the time comes.
589 break;
590 default:
Ray Milkeyf7329c72015-02-17 11:37:01 -0800591 log.warn("Unknown flow operation operator: {}", op.operator());
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800592 }
593 }
594 );
595
596 DeviceId deviceId = event.deviceId();
Sho SHIMIZU8efc8962016-08-31 15:17:44 -0700597 FlowRuleBatchOperation batchOperation = request.asBatchOperation(deviceId);
Yuta HIGUCHI48dcd372017-02-27 22:55:53 -0800598 // getProvider is customized to favor driverProvider
Thomas Vachuska27bee092015-06-23 19:03:10 -0700599 FlowRuleProvider flowRuleProvider = getProvider(deviceId);
600 if (flowRuleProvider != null) {
601 flowRuleProvider.executeBatch(batchOperation);
602 }
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800603
Yuta HIGUCHI2fcb40c2014-11-03 14:39:10 -0800604 break;
Madan Jampani117aaae2014-10-23 10:04:05 -0700605
Madan Jampani117aaae2014-10-23 10:04:05 -0700606 case BATCH_OPERATION_COMPLETED:
Yuta HIGUCHI2fcb40c2014-11-03 14:39:10 -0800607
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800608 FlowOperationsProcessor fops = pendingFlowOperations.remove(
609 event.subject().batchId());
610 if (event.result().isSuccess()) {
611 if (fops != null) {
612 fops.satisfy(event.deviceId());
613 }
614 } else {
615 fops.fail(event.deviceId(), event.result().failedItems());
616 }
617
Yuta HIGUCHIf3d51bd2014-10-21 01:05:33 -0700618 break;
Yuta HIGUCHI2fcb40c2014-11-03 14:39:10 -0800619
Yuta HIGUCHIf3d51bd2014-10-21 01:05:33 -0700620 default:
621 break;
622 }
tomc78acee2014-09-24 15:16:55 -0700623 }
624 }
alshabib902d41b2014-10-07 16:52:05 -0700625
Sho SHIMIZU5711ce12016-08-31 13:57:12 -0700626 private static FlowRuleBatchEntry.FlowRuleOperation mapOperationType(FlowRuleOperation.Type input) {
627 switch (input) {
628 case ADD:
629 return FlowRuleBatchEntry.FlowRuleOperation.ADD;
630 case MODIFY:
631 return FlowRuleBatchEntry.FlowRuleOperation.MODIFY;
632 case REMOVE:
633 return FlowRuleBatchEntry.FlowRuleOperation.REMOVE;
634 default:
635 throw new UnsupportedOperationException("Unknown flow rule type " + input);
636 }
637 }
638
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800639 private class FlowOperationsProcessor implements Runnable {
Sho SHIMIZU5f709422016-09-07 09:54:46 -0700640 // Immutable
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800641 private final FlowRuleOperations fops;
alshabib902d41b2014-10-07 16:52:05 -0700642
Sho SHIMIZU5f709422016-09-07 09:54:46 -0700643 // Mutable
644 private final List<Set<FlowRuleOperation>> stages;
Sho SHIMIZUc0bfe7c2016-09-14 16:54:30 -0700645 private final Set<DeviceId> pendingDevices = new HashSet<>();
Sho SHIMIZU5f709422016-09-07 09:54:46 -0700646 private boolean hasFailed = false;
alshabib902d41b2014-10-07 16:52:05 -0700647
Sho SHIMIZU7c9b73a2016-08-30 14:08:28 -0700648 FlowOperationsProcessor(FlowRuleOperations ops) {
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800649 this.stages = Lists.newArrayList(ops.stages());
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800650 this.fops = ops;
alshabib902d41b2014-10-07 16:52:05 -0700651 }
652
653 @Override
Sho SHIMIZU2d7c5392016-08-30 14:14:39 -0700654 public synchronized void run() {
Jon Hallcbd1b392017-01-18 20:15:44 -0800655 if (!stages.isEmpty()) {
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800656 process(stages.remove(0));
Sho SHIMIZUad4f2cd2016-09-01 13:05:56 -0700657 } else if (!hasFailed) {
Sho SHIMIZUc9e4bb02016-09-01 12:43:39 -0700658 fops.callback().onSuccess(fops);
alshabib193525b2014-10-08 18:58:03 -0700659 }
660 }
661
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800662 private void process(Set<FlowRuleOperation> ops) {
Sho SHIMIZU8efc8962016-08-31 15:17:44 -0700663 Multimap<DeviceId, FlowRuleBatchEntry> perDeviceBatches = ArrayListMultimap.create();
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800664
Sho SHIMIZUf4fd3de2016-08-31 15:47:56 -0700665 for (FlowRuleOperation op : ops) {
666 perDeviceBatches.put(op.rule().deviceId(),
667 new FlowRuleBatchEntry(mapOperationType(op.type()), op.rule()));
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800668 }
Brian O'Connorc9b64dc2016-09-13 23:01:07 +0000669 pendingDevices.addAll(perDeviceBatches.keySet());
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800670
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800671 for (DeviceId deviceId : perDeviceBatches.keySet()) {
Sho SHIMIZU3a704312015-05-27 13:36:01 -0700672 long id = idGenerator.getNewId();
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800673 final FlowRuleBatchOperation b = new FlowRuleBatchOperation(perDeviceBatches.get(deviceId),
674 deviceId, id);
Brian O'Connorc9b64dc2016-09-13 23:01:07 +0000675 pendingFlowOperations.put(id, this);
HIGUCHI Yutad9e01052016-04-14 09:31:42 -0700676 deviceInstallers.execute(() -> store.storeBatch(b));
alshabib193525b2014-10-08 18:58:03 -0700677 }
678 }
679
Sho SHIMIZU2d7c5392016-08-30 14:14:39 -0700680 synchronized void satisfy(DeviceId devId) {
Brian O'Connorc9b64dc2016-09-13 23:01:07 +0000681 pendingDevices.remove(devId);
682 if (pendingDevices.isEmpty()) {
Sho SHIMIZUc0bfe7c2016-09-14 16:54:30 -0700683 operationsService.execute(this);
alshabib193525b2014-10-08 18:58:03 -0700684 }
alshabib193525b2014-10-08 18:58:03 -0700685 }
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800686
Sho SHIMIZU2d7c5392016-08-30 14:14:39 -0700687 synchronized void fail(DeviceId devId, Set<? extends FlowRule> failures) {
Sho SHIMIZUb9e0ab72016-09-14 16:54:13 -0700688 hasFailed = true;
Brian O'Connorc9b64dc2016-09-13 23:01:07 +0000689 pendingDevices.remove(devId);
690 if (pendingDevices.isEmpty()) {
Sho SHIMIZUc0bfe7c2016-09-14 16:54:30 -0700691 operationsService.execute(this);
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800692 }
693
Sho SHIMIZUad4f2cd2016-09-01 13:05:56 -0700694 FlowRuleOperations.Builder failedOpsBuilder = FlowRuleOperations.builder();
695 failures.forEach(failedOpsBuilder::add);
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800696
Sho SHIMIZUad4f2cd2016-09-01 13:05:56 -0700697 fops.callback().onError(failedOpsBuilder.build());
Brian O'Connor72cb19a2015-01-16 16:14:41 -0800698 }
alshabib902d41b2014-10-07 16:52:05 -0700699 }
Srikanth Vavilapalli95810f52015-09-14 15:49:56 -0700700
701 @Override
702 public Iterable<TableStatisticsEntry> getFlowTableStatistics(DeviceId deviceId) {
703 checkPermission(FLOWRULE_READ);
704 return store.getTableStatistics(deviceId);
705 }
Charles Chan0c7c43b2016-01-14 17:39:20 -0800706
707 private class InternalDeviceListener implements DeviceListener {
708 @Override
709 public void event(DeviceEvent event) {
710 switch (event.type()) {
711 case DEVICE_REMOVED:
712 case DEVICE_AVAILABILITY_CHANGED:
713 DeviceId deviceId = event.subject().id();
714 if (!deviceService.isAvailable(deviceId)) {
715 if (purgeOnDisconnection) {
716 store.purgeFlowRule(deviceId);
717 }
718 }
719 break;
720 default:
721 break;
722 }
723 }
724 }
alshabib57044ba2014-09-16 15:58:01 -0700725}