Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 1 | package net.onrc.onos.apps.segmentrouting; |
| 2 | |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 3 | import java.io.IOException; |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 4 | import java.net.Inet4Address; |
| 5 | import java.net.InetAddress; |
| 6 | import java.net.UnknownHostException; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 7 | import java.util.ArrayList; |
| 8 | import java.util.Collection; |
Srikanth Vavilapalli | 363f1dc | 2014-09-22 14:30:23 -0700 | [diff] [blame] | 9 | import java.util.HashMap; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 10 | import java.util.Iterator; |
| 11 | import java.util.List; |
| 12 | import java.util.Map; |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 13 | import java.util.concurrent.ScheduledExecutorService; |
| 14 | import java.util.concurrent.TimeUnit; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 15 | |
| 16 | import net.floodlightcontroller.core.IFloodlightProviderService; |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 17 | import net.floodlightcontroller.core.IOF13Switch; |
Sangho Shin | 0df0198 | 2014-09-25 17:11:18 -0700 | [diff] [blame] | 18 | import net.floodlightcontroller.core.IOF13Switch.NeighborSet; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 19 | import net.floodlightcontroller.core.module.FloodlightModuleContext; |
| 20 | import net.floodlightcontroller.core.module.FloodlightModuleException; |
| 21 | import net.floodlightcontroller.core.module.IFloodlightModule; |
| 22 | import net.floodlightcontroller.core.module.IFloodlightService; |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 23 | import net.floodlightcontroller.core.util.SingletonTask; |
| 24 | import net.floodlightcontroller.threadpool.IThreadPoolService; |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 25 | import net.onrc.onos.api.packet.IPacketListener; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 26 | import net.onrc.onos.api.packet.IPacketService; |
| 27 | import net.onrc.onos.core.flowprogrammer.IFlowPusherService; |
| 28 | import net.onrc.onos.core.main.config.IConfigInfoService; |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 29 | import net.onrc.onos.core.matchaction.MatchAction; |
| 30 | import net.onrc.onos.core.matchaction.MatchActionId; |
| 31 | import net.onrc.onos.core.matchaction.MatchActionOperationEntry; |
| 32 | import net.onrc.onos.core.matchaction.action.Action; |
| 33 | import net.onrc.onos.core.matchaction.action.CopyTtlInAction; |
| 34 | import net.onrc.onos.core.matchaction.action.CopyTtlOutAction; |
| 35 | import net.onrc.onos.core.matchaction.action.DecMplsTtlAction; |
| 36 | import net.onrc.onos.core.matchaction.action.DecNwTtlAction; |
| 37 | import net.onrc.onos.core.matchaction.action.GroupAction; |
| 38 | import net.onrc.onos.core.matchaction.action.PopMplsAction; |
| 39 | import net.onrc.onos.core.matchaction.action.PushMplsAction; |
| 40 | import net.onrc.onos.core.matchaction.action.SetMplsIdAction; |
Saurav Das | fc5e3eb | 2014-09-25 19:05:21 -0700 | [diff] [blame] | 41 | import net.onrc.onos.core.matchaction.match.Ipv4Match; |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 42 | import net.onrc.onos.core.matchaction.match.Match; |
| 43 | import net.onrc.onos.core.matchaction.match.MplsMatch; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 44 | import net.onrc.onos.core.packet.ARP; |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 45 | import net.onrc.onos.core.packet.Ethernet; |
| 46 | import net.onrc.onos.core.packet.IPv4; |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 47 | import net.onrc.onos.core.topology.ITopologyListener; |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 48 | import net.onrc.onos.core.topology.ITopologyService; |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 49 | import net.onrc.onos.core.topology.MutableTopology; |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 50 | import net.onrc.onos.core.topology.Port; |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 51 | import net.onrc.onos.core.topology.Switch; |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 52 | import net.onrc.onos.core.topology.TopologyEvents; |
Srikanth Vavilapalli | 363f1dc | 2014-09-22 14:30:23 -0700 | [diff] [blame] | 53 | import net.onrc.onos.core.util.Dpid; |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 54 | import net.onrc.onos.core.util.IPv4Net; |
| 55 | import net.onrc.onos.core.util.SwitchPort; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 56 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 57 | import org.json.JSONArray; |
| 58 | import org.json.JSONException; |
Saurav Das | bc594a4 | 2014-09-25 20:13:50 -0700 | [diff] [blame] | 59 | import org.projectfloodlight.openflow.types.EthType; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 60 | import org.projectfloodlight.openflow.types.IPv4Address; |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 61 | import org.projectfloodlight.openflow.util.HexString; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 62 | import org.slf4j.Logger; |
| 63 | import org.slf4j.LoggerFactory; |
| 64 | |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 65 | public class SegmentRoutingManager implements IFloodlightModule, |
Sangho Shin | ac5ee2b | 2014-09-28 21:27:20 -0700 | [diff] [blame] | 66 | ITopologyListener, IPacketListener { |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 67 | |
| 68 | private static final Logger log = LoggerFactory |
| 69 | .getLogger(SegmentRoutingManager.class); |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 70 | private ITopologyService topologyService; |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 71 | private IPacketService packetService; |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 72 | private MutableTopology mutableTopology; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 73 | |
| 74 | private List<ArpEntry> arpEntries; |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 75 | private ArpHandler arpHandler; |
| 76 | private GenericIpHandler ipHandler; |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 77 | private IcmpHandler icmpHandler; |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 78 | private boolean networkConverged; |
| 79 | private IThreadPoolService threadPool; |
| 80 | private SingletonTask discoveryTask; |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 81 | private IFloodlightProviderService floodlightProvider; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 82 | |
| 83 | @Override |
| 84 | public Collection<Class<? extends IFloodlightService>> getModuleServices() { |
| 85 | // TODO Auto-generated method stub |
| 86 | return null; |
| 87 | } |
| 88 | |
| 89 | @Override |
| 90 | public Map<Class<? extends IFloodlightService>, IFloodlightService> getServiceImpls() { |
| 91 | // TODO Auto-generated method stub |
| 92 | return null; |
| 93 | } |
| 94 | |
| 95 | @Override |
| 96 | public Collection<Class<? extends IFloodlightService>> getModuleDependencies() { |
| 97 | Collection<Class<? extends IFloodlightService>> l = new ArrayList<Class<? extends IFloodlightService>>(); |
| 98 | |
| 99 | l.add(IFloodlightProviderService.class); |
| 100 | l.add(IConfigInfoService.class); |
| 101 | l.add(ITopologyService.class); |
| 102 | l.add(IPacketService.class); |
| 103 | l.add(IFlowPusherService.class); |
| 104 | l.add(ITopologyService.class); |
| 105 | |
| 106 | return l; |
| 107 | |
| 108 | } |
| 109 | |
| 110 | @Override |
| 111 | public void init(FloodlightModuleContext context) throws FloodlightModuleException { |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 112 | floodlightProvider = context.getServiceImpl(IFloodlightProviderService.class); |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 113 | arpHandler = new ArpHandler(context, this); |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 114 | icmpHandler = new IcmpHandler(context, this); |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 115 | ipHandler = new GenericIpHandler(context, this); |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 116 | arpEntries = new ArrayList<ArpEntry>(); |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 117 | topologyService = context.getServiceImpl(ITopologyService.class); |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 118 | threadPool = context.getServiceImpl(IThreadPoolService.class); |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 119 | mutableTopology = topologyService.getTopology(); |
| 120 | topologyService.addListener(this, false); |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 121 | |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 122 | this.packetService = context.getServiceImpl(IPacketService.class); |
| 123 | packetService.registerPacketListener(this); |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 124 | |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 125 | } |
| 126 | |
| 127 | @Override |
| 128 | public void startUp(FloodlightModuleContext context) throws FloodlightModuleException { |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 129 | networkConverged = false; |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 130 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 131 | ScheduledExecutorService ses = threadPool.getScheduledExecutor(); |
| 132 | |
| 133 | discoveryTask = new SingletonTask(ses, new Runnable() { |
| 134 | @Override |
| 135 | public void run() { |
| 136 | populateEcmpRoutingRules(); |
| 137 | } |
| 138 | }); |
| 139 | |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 140 | discoveryTask.reschedule(15, TimeUnit.SECONDS); |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 141 | } |
| 142 | |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 143 | @Override |
| 144 | public void receive(Switch sw, Port inPort, Ethernet payload) { |
| 145 | if (payload.getEtherType() == Ethernet.TYPE_ARP) |
| 146 | arpHandler.processPacketIn(sw, inPort, payload); |
| 147 | if (payload.getEtherType() == Ethernet.TYPE_IPV4) { |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 148 | if (((IPv4)payload.getPayload()).getProtocol() == IPv4.PROTOCOL_ICMP) |
Srikanth Vavilapalli | b1fce73 | 2014-09-24 14:09:50 -0700 | [diff] [blame] | 149 | icmpHandler.processPacketIn(sw, inPort, payload); |
| 150 | else |
| 151 | ipHandler.processPacketIn(sw, inPort, payload); |
| 152 | } |
| 153 | } |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 154 | /** |
| 155 | * Update ARP Cache using ARP packets |
| 156 | * It is used to set destination MAC address to forward packets to known hosts. |
| 157 | * But, it will be replace with Host information of Topology service later. |
| 158 | * |
| 159 | * @param arp APR packets to use for updating ARP entries |
| 160 | */ |
| 161 | public void updateArpCache(ARP arp) { |
| 162 | |
| 163 | ArpEntry arpEntry = new ArpEntry(arp.getSenderHardwareAddress(), arp.getSenderProtocolAddress()); |
| 164 | // TODO: Need to check the duplication |
| 165 | arpEntries.add(arpEntry); |
| 166 | } |
| 167 | |
| 168 | /** |
| 169 | * Get MAC address to known hosts |
| 170 | * |
| 171 | * @param destinationAddress IP address to get MAC address |
| 172 | * @return MAC Address to given IP address |
| 173 | */ |
| 174 | public byte[] getMacAddressFromIpAddress(int destinationAddress) { |
| 175 | |
| 176 | // Can't we get the host IP address from the TopologyService ?? |
| 177 | |
| 178 | Iterator<ArpEntry> iterator = arpEntries.iterator(); |
| 179 | |
| 180 | IPv4Address ipAddress = IPv4Address.of(destinationAddress); |
| 181 | byte[] ipAddressInByte = ipAddress.getBytes(); |
| 182 | |
| 183 | while (iterator.hasNext() ) { |
| 184 | ArpEntry arpEntry = iterator.next(); |
| 185 | byte[] address = arpEntry.targetIpAddress; |
| 186 | |
| 187 | IPv4Address a = IPv4Address.of(address); |
| 188 | IPv4Address b = IPv4Address.of(ipAddressInByte); |
| 189 | |
| 190 | if ( a.equals(b)) { |
| 191 | log.debug("Found an arp entry"); |
| 192 | return arpEntry.targetMacAddress; |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | return null; |
| 197 | } |
| 198 | |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 199 | /** |
| 200 | * Send an ARP request via ArpHandler |
| 201 | * @param destinationAddress |
| 202 | * @param sw |
| 203 | * @param inPort |
| 204 | * |
| 205 | */ |
| 206 | public void sendArpRequest(Switch sw, int destinationAddress, Port inPort) { |
| 207 | arpHandler.sendArpRequest(sw, destinationAddress, inPort); |
| 208 | } |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 209 | |
| 210 | /** |
| 211 | * Temporary class to to keep ARP entry |
| 212 | * |
| 213 | */ |
| 214 | private class ArpEntry { |
| 215 | |
| 216 | byte[] targetMacAddress; |
| 217 | byte[] targetIpAddress; |
| 218 | |
| 219 | private ArpEntry(byte[] macAddress, byte[] ipAddress) { |
| 220 | this.targetMacAddress = macAddress; |
| 221 | this.targetIpAddress = ipAddress; |
| 222 | } |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 223 | } |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 224 | |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 225 | /** |
| 226 | * Topology events that have been generated. |
| 227 | * |
| 228 | * @param topologyEvents the generated Topology Events |
| 229 | * @see TopologyEvents |
| 230 | */ |
| 231 | public void topologyEvents(TopologyEvents topologyEvents) |
| 232 | { |
| 233 | /** |
| 234 | * Any Link update events, compute the ECMP path graph for all switch nodes |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 235 | |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 236 | if ((topologyEvents.getAddedLinkDataEntries() != null) || |
| 237 | (topologyEvents.getRemovedLinkDataEntries() != null)) |
| 238 | { |
| 239 | Iterable<Switch> switches= mutableTopology.getSwitches(); |
| 240 | for (Switch sw : switches) { |
| 241 | ECMPShortestPathGraph ecmpSPG = new ECMPShortestPathGraph(sw); |
Srikanth Vavilapalli | f3bfcf9 | 2014-09-19 07:42:10 -0700 | [diff] [blame] | 242 | log.debug("ECMPShortestPathGraph is computed for switch {}", |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 243 | HexString.toHexString(sw.getDpid().value())); |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 244 | |
| 245 | |
Srikanth Vavilapalli | 363f1dc | 2014-09-22 14:30:23 -0700 | [diff] [blame] | 246 | HashMap<Integer, HashMap<Switch,ArrayList<Path>>> pathGraph = |
| 247 | ecmpSPG.getCompleteLearnedSwitchesAndPaths(); |
| 248 | for (Integer itrIdx: pathGraph.keySet()){ |
| 249 | |
| 250 | HashMap<Switch, ArrayList<Path>> swPathsMap = |
| 251 | pathGraph.get(itrIdx); |
| 252 | for (Switch targetSw: swPathsMap.keySet()){ |
| 253 | log.debug("ECMPShortestPathGraph:Paths in Pass{} from " |
| 254 | + " switch {} to switch {}:****", |
| 255 | itrIdx, |
| 256 | HexString.toHexString(sw.getDpid().value()), |
| 257 | HexString.toHexString(targetSw.getDpid().value())); |
| 258 | int i=0; |
| 259 | for (Path path:swPathsMap.get(targetSw)){ |
| 260 | log.debug("****ECMPShortestPathGraph:Path{} is {}",i++,path); |
| 261 | } |
| 262 | } |
| 263 | } |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 264 | |
Srikanth Vavilapalli | 363f1dc | 2014-09-22 14:30:23 -0700 | [diff] [blame] | 265 | HashMap<Integer, HashMap<Switch,ArrayList<ArrayList<Dpid>>>> switchVia = |
| 266 | ecmpSPG.getAllLearnedSwitchesAndVia(); |
| 267 | for (Integer itrIdx: switchVia.keySet()){ |
| 268 | log.debug("ECMPShortestPathGraph:Switches learned in " |
| 269 | + "Iteration{} from switch {}:", |
| 270 | itrIdx, |
| 271 | HexString.toHexString(sw.getDpid().value())); |
| 272 | |
| 273 | HashMap<Switch, ArrayList<ArrayList<Dpid>>> swViaMap = |
| 274 | switchVia.get(itrIdx); |
| 275 | for (Switch targetSw: swViaMap.keySet()){ |
| 276 | log.debug("ECMPShortestPathGraph:****switch {} via:", |
| 277 | HexString.toHexString(targetSw.getDpid().value())); |
| 278 | int i=0; |
| 279 | for (ArrayList<Dpid> via:swViaMap.get(targetSw)){ |
| 280 | log.debug("ECMPShortestPathGraph:******{}) {}",++i,via); |
| 281 | } |
| 282 | } |
| 283 | } |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 284 | |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 285 | } |
| 286 | } |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 287 | */ |
| 288 | |
| 289 | if ((topologyEvents.getAddedLinkDataEntries() != null) || |
| 290 | (topologyEvents.getRemovedLinkDataEntries() != null)) |
| 291 | { |
| 292 | |
| 293 | if (networkConverged) { |
| 294 | populateEcmpRoutingRules(); |
| 295 | } |
| 296 | } |
| 297 | |
Srikanth Vavilapalli | b7e5c5e | 2014-09-18 07:38:27 -0700 | [diff] [blame] | 298 | } |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 299 | |
| 300 | /** |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 301 | * Populate routing rules walking through the ECMP shortest paths |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 302 | * |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 303 | */ |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 304 | private void populateEcmpRoutingRules() { |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 305 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 306 | Iterable<Switch> switches= mutableTopology.getSwitches(); |
| 307 | for (Switch sw : switches) { |
| 308 | ECMPShortestPathGraph ecmpSPG = new ECMPShortestPathGraph(sw); |
| 309 | log.debug("ECMPShortestPathGraph is computed for switch {}", |
| 310 | HexString.toHexString(sw.getDpid().value())); |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 311 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 312 | HashMap<Integer, HashMap<Switch,ArrayList<ArrayList<Dpid>>>> switchVia = |
| 313 | ecmpSPG.getAllLearnedSwitchesAndVia(); |
| 314 | for (Integer itrIdx: switchVia.keySet()){ |
| 315 | log.debug("ECMPShortestPathGraph:Switches learned in " |
| 316 | + "Iteration{} from switch {}:", |
| 317 | itrIdx, |
| 318 | HexString.toHexString(sw.getDpid().value())); |
| 319 | HashMap<Switch, ArrayList<ArrayList<Dpid>>> swViaMap = |
| 320 | switchVia.get(itrIdx); |
| 321 | for (Switch targetSw: swViaMap.keySet()){ |
| 322 | log.debug("ECMPShortestPathGraph:****switch {} via:", |
| 323 | HexString.toHexString(targetSw.getDpid().value())); |
| 324 | String destSw = sw.getDpid().toString(); |
| 325 | List<String> fwdToSw = new ArrayList<String>(); |
| 326 | |
| 327 | int i=0; |
| 328 | for (ArrayList<Dpid> via:swViaMap.get(targetSw)){ |
| 329 | log.debug("ECMPShortestPathGraph:******{}) {}",++i,via); |
| 330 | if (via.isEmpty()) { |
| 331 | fwdToSw.add(destSw); |
| 332 | } |
| 333 | else { |
| 334 | fwdToSw.add(via.get(0).toString()); |
| 335 | } |
| 336 | } |
| 337 | setRoutingRule(targetSw, destSw, fwdToSw); |
| 338 | } |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 339 | } |
| 340 | } |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 341 | |
| 342 | networkConverged = true; |
| 343 | } |
| 344 | |
| 345 | /** |
| 346 | * |
| 347 | * Set routing rules in targetSw |
| 348 | * {forward packets to fwdToSw switches in order to send packets to destSw} |
| 349 | * - If the target switch is an edge router and final destnation switch is also |
| 350 | * an edge router, then set IP forwarding rules to subnets |
| 351 | * - If only the target switch is an edge router, then set IP forwarding rule to |
| 352 | * the transit router loopback IP address |
| 353 | * - If the target is a transit router, then just set the MPLS forwarding rule |
| 354 | * |
| 355 | * @param targetSw Switch to set the rules |
| 356 | * @param destSw Final destination switches |
| 357 | * @param fwdToSw next hop switches |
| 358 | */ |
| 359 | private void setRoutingRule(Switch targetSw, String destSw, List<String> fwdToSw) { |
| 360 | |
| 361 | |
| 362 | if (fwdToSw.isEmpty()) { |
| 363 | fwdToSw.add(destSw); |
| 364 | } |
| 365 | |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 366 | // if both target SW and dest SW are an edge router, then set IP table |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 367 | if (IsEdgeRouter(targetSw.getDpid().toString()) && |
| 368 | IsEdgeRouter(destSw)) { |
| 369 | // We assume that there is at least one transit router b/w edge routers |
| 370 | Switch destSwitch = mutableTopology.getSwitch(new Dpid(destSw)); |
| 371 | String subnets = destSwitch.getStringAttribute("subnets"); |
| 372 | try { |
| 373 | JSONArray arry = new JSONArray(subnets); |
| 374 | for (int i = 0; i < arry.length(); i++) { |
| 375 | String subnetIp = (String) arry.getJSONObject(i).get("subnetIp"); |
| 376 | setIpTableRouter(targetSw, subnetIp, getMplsLabel(destSw) |
| 377 | ,fwdToSw); |
| 378 | |
| 379 | } |
| 380 | } catch (JSONException e) { |
| 381 | // TODO Auto-generated catch block |
| 382 | e.printStackTrace(); |
| 383 | } |
| 384 | String routerIp = destSwitch.getStringAttribute("routerIp"); |
| 385 | setIpTableRouter(targetSw, routerIp, getMplsLabel(destSw), fwdToSw); |
| 386 | } |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 387 | // Only if the target switch is the edge router, then set the IP rules |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 388 | else if (IsEdgeRouter(targetSw.getDpid().toString())) { |
| 389 | // We assume that there is at least one transit router b/w edge routers |
| 390 | Switch destSwitch = mutableTopology.getSwitch(new Dpid(destSw)); |
| 391 | String routerIp = destSwitch.getStringAttribute("routerIp"); |
| 392 | setIpTableRouter(targetSw, routerIp, getMplsLabel(destSw), fwdToSw); |
| 393 | } |
| 394 | // if it is a transit router, then set rules in the MPLS table |
| 395 | else { |
| 396 | setMplsTable(targetSw, getMplsLabel(destSw), fwdToSw); |
| 397 | } |
| 398 | |
| 399 | } |
| 400 | |
| 401 | /** |
| 402 | * Check if the switch is the edge router or not |
| 403 | * If any subnet information is defined in the config file, the we assume |
| 404 | * it is an edge router |
| 405 | * |
| 406 | * @param dpid Dpid of the switch to check |
| 407 | * @return true if it is an edge router, otherwise false |
| 408 | */ |
| 409 | private boolean IsEdgeRouter(String dpid) { |
| 410 | |
| 411 | for (Switch sw: mutableTopology.getSwitches()) { |
| 412 | String dpidStr = sw.getDpid().toString(); |
| 413 | if (dpid.equals(dpidStr)) { |
| 414 | String subnetInfo = sw.getStringAttribute("subnets"); |
| 415 | if (subnetInfo == null || subnetInfo.equals("[]")) { |
| 416 | return false; |
| 417 | } |
| 418 | else |
| 419 | return true; |
| 420 | } |
| 421 | } |
| 422 | |
| 423 | return false; |
| 424 | } |
| 425 | |
| 426 | /** |
| 427 | * Set IP forwarding rule |
| 428 | * - If the destination is the next hop, then do not push MPLS, |
| 429 | * just decrease the NW TTL |
| 430 | * - Otherwise, push MPLS label and set the MPLS ID |
| 431 | * |
| 432 | * @param sw target switch to set rules |
| 433 | * @param subnetIp Match IP address |
| 434 | * @param mplsLabel MPLS label of final destination router |
| 435 | * @param fwdToSws next hop routers |
| 436 | */ |
| 437 | private void setIpTableRouter(Switch sw, String subnetIp, String mplsLabel, |
| 438 | List<String> fwdToSws) { |
| 439 | |
Saurav Das | fc5e3eb | 2014-09-25 19:05:21 -0700 | [diff] [blame] | 440 | Ipv4Match ipMatch = new Ipv4Match(subnetIp); |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 441 | List<Action> actions = new ArrayList<>(); |
| 442 | |
| 443 | // If destination SW is the same as the fwd SW, then do not push MPLS label |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 444 | |
| 445 | if (fwdToSws.size() > 1) { |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 446 | PushMplsAction pushMplsAction = new PushMplsAction(); |
| 447 | SetMplsIdAction setIdAction = new SetMplsIdAction(Integer.parseInt(mplsLabel)); |
| 448 | CopyTtlOutAction copyTtlOutAction = new CopyTtlOutAction(); |
| 449 | |
| 450 | actions.add(pushMplsAction); |
| 451 | actions.add(setIdAction); |
| 452 | actions.add(copyTtlOutAction); |
| 453 | } |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 454 | else { |
| 455 | String fwdToSw = fwdToSws.get(0); |
| 456 | if (getMplsLabel(fwdToSw).equals(mplsLabel)) { |
| 457 | DecNwTtlAction decTtlAction = new DecNwTtlAction(1); |
| 458 | actions.add(decTtlAction); |
| 459 | } |
| 460 | else { |
| 461 | PushMplsAction pushMplsAction = new PushMplsAction(); |
| 462 | SetMplsIdAction setIdAction = new SetMplsIdAction(Integer.parseInt(mplsLabel)); |
| 463 | CopyTtlOutAction copyTtlOutAction = new CopyTtlOutAction(); |
| 464 | |
| 465 | actions.add(pushMplsAction); |
| 466 | actions.add(setIdAction); |
| 467 | actions.add(copyTtlOutAction); |
| 468 | } |
| 469 | } |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 470 | |
| 471 | GroupAction groupAction = new GroupAction(); |
| 472 | |
| 473 | for (String fwdSw : fwdToSws) { |
| 474 | groupAction.addSwitch(new Dpid(fwdSw)); |
| 475 | } |
| 476 | actions.add(groupAction); |
| 477 | |
| 478 | //MatchAction matchAction = new MatchAction(maIdGenerator.getNewId(), |
| 479 | MatchAction matchAction = new MatchAction(new MatchActionId(0), |
| 480 | new SwitchPort((long)0,(short)0), ipMatch, actions); |
| 481 | |
| 482 | MatchActionOperationEntry maEntry = |
| 483 | new MatchActionOperationEntry( |
| 484 | net.onrc.onos.core.matchaction.MatchActionOperations.Operator.ADD, |
| 485 | matchAction); |
| 486 | |
Sangho Shin | ac5ee2b | 2014-09-28 21:27:20 -0700 | [diff] [blame] | 487 | IOF13Switch sw13 = (IOF13Switch)floodlightProvider.getMasterSwitch( |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 488 | getSwId(sw.getDpid().toString())); |
| 489 | |
| 490 | try { |
| 491 | printMatchActionOperationEntry(sw, maEntry); |
| 492 | sw13.pushFlow(maEntry); |
| 493 | } catch (IOException e) { |
| 494 | // TODO Auto-generated catch block |
| 495 | e.printStackTrace(); |
| 496 | } |
| 497 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 498 | } |
| 499 | |
| 500 | |
Sangho Shin | ac5ee2b | 2014-09-28 21:27:20 -0700 | [diff] [blame] | 501 | /** |
| 502 | * Convert a string DPID to its Switch Id (integer) |
| 503 | * |
| 504 | * @param dpid |
| 505 | * @return |
| 506 | */ |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 507 | private long getSwId(String dpid) { |
| 508 | |
| 509 | long swId = 0; |
| 510 | |
| 511 | String swIdStr = dpid.substring(dpid.lastIndexOf(":")+1); |
| 512 | if (swIdStr != null) |
| 513 | swId = Integer.parseInt(swIdStr); |
| 514 | |
| 515 | return swId; |
| 516 | } |
| 517 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 518 | /** |
| 519 | * Set MPLS forwarding rules to MPLS table |
| 520 | * - If the destination is the same as the next hop to forward packets |
| 521 | * then, pop the MPLS label according to PHP rule |
| 522 | * - Otherwise, just forward packets to next hops using Group action |
| 523 | * |
| 524 | * @param sw Switch to set the rules |
| 525 | * @param mplsLabel destination MPLS label |
| 526 | * @param fwdSws next hop switches |
| 527 | */ |
| 528 | private void setMplsTable(Switch sw, String mplsLabel, List<String> fwdSws) { |
Saurav Das | d097744 | 2014-09-29 10:36:14 -0700 | [diff] [blame^] | 529 | if (mplsLabel == null) { |
| 530 | log.error("mpls label not configured for sw: {}. Not populating" |
| 531 | + " MPLS table entries.", sw.getDpid()); |
| 532 | return; |
| 533 | } |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 534 | MplsMatch mplsMatch = new MplsMatch(Integer.parseInt(mplsLabel)); |
| 535 | |
| 536 | List<Action> actions = new ArrayList<Action>(); |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 537 | |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 538 | if (fwdSws.size() > 1) { |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 539 | DecMplsTtlAction decMplsTtlAction = new DecMplsTtlAction(1); |
| 540 | actions.add(decMplsTtlAction); |
| 541 | } |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 542 | // If the destination is the same as the next hop, then pop MPLS |
| 543 | else { |
| 544 | String fwdMplsId = getMplsLabel(fwdSws.get(0)); |
| 545 | if (fwdMplsId.equals(mplsLabel)) { |
| 546 | String fwdSw = fwdSws.get(0); |
| 547 | if (mplsLabel.equals(getMplsLabel(fwdSw))) { |
| 548 | PopMplsAction popAction = new PopMplsAction(EthType.IPv4); |
| 549 | CopyTtlInAction copyTtlInAction = new CopyTtlInAction(); |
| 550 | |
| 551 | actions.add(popAction); |
| 552 | actions.add(copyTtlInAction); |
| 553 | } |
| 554 | } |
| 555 | } |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 556 | |
| 557 | GroupAction groupAction = new GroupAction(); |
| 558 | for (String fwdSw: fwdSws) |
| 559 | groupAction.addSwitch(new Dpid(fwdSw)); |
| 560 | actions.add(groupAction); |
| 561 | |
| 562 | MatchAction matchAction = new MatchAction(new MatchActionId(0), |
| 563 | new SwitchPort((long)0,(short)0), mplsMatch, actions); |
| 564 | |
| 565 | MatchActionOperationEntry maEntry = |
| 566 | new MatchActionOperationEntry( |
| 567 | net.onrc.onos.core.matchaction.MatchActionOperations.Operator.ADD, |
| 568 | matchAction); |
| 569 | |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 570 | IOF13Switch sw13 = (IOF13Switch)floodlightProvider.getMasterSwitch( |
| 571 | getSwId(sw.getDpid().toString())); |
| 572 | |
| 573 | try { |
| 574 | printMatchActionOperationEntry(sw, maEntry); |
| 575 | sw13.pushFlow(maEntry); |
| 576 | } catch (IOException e) { |
| 577 | // TODO Auto-generated catch block |
| 578 | e.printStackTrace(); |
| 579 | } |
| 580 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 581 | } |
| 582 | |
| 583 | |
| 584 | /** |
| 585 | * Debugging function to print out the Match Action Entry |
| 586 | * |
| 587 | * @param maEntry |
| 588 | */ |
Sangho Shin | ac5ee2b | 2014-09-28 21:27:20 -0700 | [diff] [blame] | 589 | private void printMatchActionOperationEntry(Switch sw, |
| 590 | MatchActionOperationEntry maEntry) { |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 591 | |
Sangho Shin | 0df0198 | 2014-09-25 17:11:18 -0700 | [diff] [blame] | 592 | StringBuilder logStr = new StringBuilder("In switch " + sw.getDpid() + ", "); |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 593 | |
| 594 | MatchAction ma = maEntry.getTarget(); |
| 595 | Match m = ma.getMatch(); |
| 596 | List<Action> actions = ma.getActions(); |
| 597 | |
Saurav Das | fc5e3eb | 2014-09-25 19:05:21 -0700 | [diff] [blame] | 598 | if (m instanceof Ipv4Match) { |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 599 | logStr.append("If the IP matches with "); |
Saurav Das | fc5e3eb | 2014-09-25 19:05:21 -0700 | [diff] [blame] | 600 | IPv4Net ip = ((Ipv4Match) m).getDestination(); |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 601 | logStr.append(ip.toString()); |
| 602 | logStr.append(" then "); |
| 603 | } |
| 604 | else if (m instanceof MplsMatch) { |
| 605 | logStr.append("If the MPLS label matches with "); |
| 606 | int mplsLabel = ((MplsMatch) m).getMplsLabel(); |
| 607 | logStr.append(mplsLabel); |
| 608 | logStr.append(" then "); |
| 609 | } |
| 610 | |
| 611 | logStr.append(" do { "); |
| 612 | for (Action action: actions) { |
| 613 | if (action instanceof CopyTtlInAction) { |
| 614 | logStr.append("copy ttl In, "); |
| 615 | } |
| 616 | else if (action instanceof CopyTtlOutAction) { |
| 617 | logStr.append("copy ttl Out, "); |
| 618 | } |
| 619 | else if (action instanceof DecMplsTtlAction) { |
| 620 | logStr.append("Dec MPLS TTL , "); |
| 621 | } |
| 622 | else if (action instanceof GroupAction) { |
| 623 | logStr.append("Forward packet to < "); |
Sangho Shin | 0df0198 | 2014-09-25 17:11:18 -0700 | [diff] [blame] | 624 | NeighborSet dpids = ((GroupAction)action).getDpids(); |
| 625 | logStr.append(dpids.toString() + ","); |
| 626 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 627 | } |
| 628 | else if (action instanceof PopMplsAction) { |
| 629 | logStr.append("Pop MPLS label, "); |
| 630 | } |
| 631 | else if (action instanceof PushMplsAction) { |
| 632 | logStr.append("Push MPLS label, "); |
| 633 | } |
| 634 | else if (action instanceof SetMplsIdAction) { |
| 635 | int id = ((SetMplsIdAction)action).getMplsId(); |
| 636 | logStr.append("Set MPLS ID as " + id + ", "); |
| 637 | |
| 638 | } |
| 639 | } |
| 640 | |
| 641 | log.debug(logStr.toString()); |
| 642 | |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 643 | } |
| 644 | |
| 645 | /** |
| 646 | * Get MPLS label reading the config file |
| 647 | * |
| 648 | * @param dipid DPID of the switch |
| 649 | * @return MPLS label for the switch |
| 650 | */ |
| 651 | |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 652 | private String getMplsLabel(String dpid) { |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 653 | |
| 654 | String mplsLabel = null; |
| 655 | for (Switch sw: mutableTopology.getSwitches()) { |
Sangho Shin | 43cee11 | 2014-09-25 16:43:34 -0700 | [diff] [blame] | 656 | String dpidStr = sw.getDpid().toString(); |
| 657 | if (dpid.equals(dpidStr)) { |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 658 | mplsLabel = sw.getStringAttribute("nodeSid"); |
| 659 | break; |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 660 | } |
| 661 | } |
| 662 | |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 663 | return mplsLabel; |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 664 | } |
| 665 | |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 666 | |
| 667 | |
| 668 | /** |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 669 | * The function checks if given IP matches to the given subnet mask |
| 670 | * |
| 671 | * @param addr - subnet address to match |
| 672 | * @param addr1 - IP address to check |
| 673 | * @return true if the IP address matches to the subnet, otherwise false |
| 674 | */ |
Sangho Shin | 1aa9354 | 2014-09-22 09:49:44 -0700 | [diff] [blame] | 675 | public boolean netMatch(String addr, String addr1){ //addr is subnet address and addr1 is ip address. Function will return true, if addr1 is within addr(subnet) |
| 676 | |
| 677 | String[] parts = addr.split("/"); |
| 678 | String ip = parts[0]; |
| 679 | int prefix; |
| 680 | |
| 681 | if (parts.length < 2) { |
| 682 | prefix = 0; |
| 683 | } else { |
| 684 | prefix = Integer.parseInt(parts[1]); |
| 685 | } |
| 686 | |
| 687 | Inet4Address a =null; |
| 688 | Inet4Address a1 =null; |
| 689 | try { |
| 690 | a = (Inet4Address) InetAddress.getByName(ip); |
| 691 | a1 = (Inet4Address) InetAddress.getByName(addr1); |
| 692 | } catch (UnknownHostException e){} |
| 693 | |
| 694 | byte[] b = a.getAddress(); |
| 695 | int ipInt = ((b[0] & 0xFF) << 24) | |
| 696 | ((b[1] & 0xFF) << 16) | |
| 697 | ((b[2] & 0xFF) << 8) | |
| 698 | ((b[3] & 0xFF) << 0); |
| 699 | |
| 700 | byte[] b1 = a1.getAddress(); |
| 701 | int ipInt1 = ((b1[0] & 0xFF) << 24) | |
| 702 | ((b1[1] & 0xFF) << 16) | |
| 703 | ((b1[2] & 0xFF) << 8) | |
| 704 | ((b1[3] & 0xFF) << 0); |
| 705 | |
| 706 | int mask = ~((1 << (32 - prefix)) - 1); |
| 707 | |
| 708 | if ((ipInt & mask) == (ipInt1 & mask)) { |
| 709 | return true; |
| 710 | } |
| 711 | else { |
| 712 | return false; |
| 713 | } |
| 714 | } |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 715 | |
Sangho Shin | ac5ee2b | 2014-09-28 21:27:20 -0700 | [diff] [blame] | 716 | /** |
| 717 | * Add a routing rule for the host |
| 718 | * |
| 719 | * @param sw - Switch to add the rule |
| 720 | * @param hostIpAddress Destination host IP address |
| 721 | * @param hostMacAddress Destination host MAC address |
| 722 | */ |
Sangho Shin | eb08303 | 2014-09-22 16:11:34 -0700 | [diff] [blame] | 723 | public void addRouteToHost(Switch sw, int hostIpAddress, byte[] hostMacAddress) { |
| 724 | ipHandler.addRouteToHost(sw, hostIpAddress, hostMacAddress); |
| 725 | |
| 726 | } |
Sangho Shin | 9c0f4c3 | 2014-09-26 16:02:38 -0700 | [diff] [blame] | 727 | |
Sangho Shin | 2f26369 | 2014-09-15 14:09:41 -0700 | [diff] [blame] | 728 | } |