blob: 5df07b66d504646502e4664096918156531b96e2 [file] [log] [blame]
Carmelo Cascone3c216fa2018-06-22 14:52:15 +02001import multiprocessing
Carmelo Casconeb7524272017-06-05 16:53:13 -04002import os
Carmelo Cascone15693a22018-12-12 19:06:57 -08003
4import json
Carmelo Cascone44448a52018-06-25 23:36:57 +02005import random
Carmelo Casconeb7524272017-06-05 16:53:13 -04006import re
Carmelo Cascone44448a52018-06-25 23:36:57 +02007import socket
Carmelo Casconef11513d2018-01-16 00:31:14 -08008import threading
Carmelo Cascone15693a22018-12-12 19:06:57 -08009import time
Carmelo Cascone44448a52018-06-25 23:36:57 +020010import urllib2
Carmelo Casconef11513d2018-01-16 00:31:14 -080011from contextlib import closing
Carmelo Cascone44448a52018-06-25 23:36:57 +020012from mininet.log import info, warn
Carmelo Cascone34433252017-08-25 20:27:18 +020013from mininet.node import Switch, Host
Carmelo Cascone785fada2016-06-16 18:34:16 -070014
Carmelo Casconef11513d2018-01-16 00:31:14 -080015SIMPLE_SWITCH_GRPC = 'simple_switch_grpc'
Carmelo Cascone34433252017-08-25 20:27:18 +020016PKT_BYTES_TO_DUMP = 80
Carmelo Cascone46d360b2017-08-29 20:20:32 +020017VALGRIND_PREFIX = 'valgrind --leak-check=yes'
Carmelo Casconef11513d2018-01-16 00:31:14 -080018SWITCH_START_TIMEOUT = 5 # seconds
19BMV2_LOG_LINES = 5
Carmelo Cascone03ae0ac2018-10-11 08:31:59 -070020BMV2_DEFAULT_DEVICE_ID = 1
Carmelo Casconef11513d2018-01-16 00:31:14 -080021
Carmelo Cascone46d360b2017-08-29 20:20:32 +020022
23def parseBoolean(value):
24 if value in ['1', 1, 'true', 'True']:
25 return True
26 else:
27 return False
Carmelo Cascone34433252017-08-25 20:27:18 +020028
29
Carmelo Casconef11513d2018-01-16 00:31:14 -080030def pickUnusedPort():
31 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
32 s.bind(('localhost', 0))
33 addr, port = s.getsockname()
34 s.close()
35 return port
36
37
38def writeToFile(path, value):
39 with open(path, "w") as f:
40 f.write(str(value))
41
42
43def watchDog(sw):
44 while True:
Carmelo Cascone3c216fa2018-06-22 14:52:15 +020045 if ONOSBmv2Switch.mininet_exception == 1:
46 sw.killBmv2(log=False)
47 return
Carmelo Casconef11513d2018-01-16 00:31:14 -080048 if sw.stopped:
49 return
50 with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
51 if s.connect_ex(('127.0.0.1', sw.grpcPort)) == 0:
52 time.sleep(1)
53 else:
Carmelo Casconedf32bea2018-06-19 17:45:54 +020054 warn("\n*** WARN: BMv2 instance %s died!\n" % sw.name)
Carmelo Casconef11513d2018-01-16 00:31:14 -080055 sw.printBmv2Log()
56 print ("-" * 80) + "\n"
57 return
58
59
Carmelo Cascone34433252017-08-25 20:27:18 +020060class ONOSHost(Host):
61 def __init__(self, name, inNamespace=True, **params):
62 Host.__init__(self, name, inNamespace=inNamespace, **params)
63
64 def config(self, **params):
65 r = super(Host, self).config(**params)
66 for off in ["rx", "tx", "sg"]:
Carmelo Casconef11513d2018-01-16 00:31:14 -080067 cmd = "/sbin/ethtool --offload %s %s off" \
Carmelo Cascone6ec8f8f2017-11-22 14:27:06 -080068 % (self.defaultIntf(), off)
Carmelo Cascone34433252017-08-25 20:27:18 +020069 self.cmd(cmd)
70 # disable IPv6
71 self.cmd("sysctl -w net.ipv6.conf.all.disable_ipv6=1")
72 self.cmd("sysctl -w net.ipv6.conf.default.disable_ipv6=1")
73 self.cmd("sysctl -w net.ipv6.conf.lo.disable_ipv6=1")
74 return r
Carmelo Casconeb7524272017-06-05 16:53:13 -040075
Carmelo Cascone785fada2016-06-16 18:34:16 -070076
77class ONOSBmv2Switch(Switch):
Carmelo Casconeb7524272017-06-05 16:53:13 -040078 """BMv2 software switch with gRPC server"""
Carmelo Cascone3c216fa2018-06-22 14:52:15 +020079 # Shared value used to notify to all instances of this class that a Mininet
80 # exception occurred. Mininet exception handling doesn't call the stop()
81 # method, so the mn process would hang after clean-up since Bmv2 would still
82 # be running.
83 mininet_exception = multiprocessing.Value('i', 0)
Carmelo Cascone785fada2016-06-16 18:34:16 -070084
Carmelo Casconef11513d2018-01-16 00:31:14 -080085 def __init__(self, name, json=None, debugger=False, loglevel="warn",
Carmelo Cascone76e63862018-09-04 14:25:49 -070086 elogger=False, grpcport=None, cpuport=255, notifications=False,
Carmelo Casconeeaa8b1d2018-04-11 14:12:17 -070087 thriftport=None, netcfg=True, dryrun=False, pipeconf="",
88 pktdump=False, valgrind=False, gnmi=False,
Carmelo Cascone55965c62018-05-17 18:13:16 -070089 portcfg=True, onosdevid=None, **kwargs):
Carmelo Cascone785fada2016-06-16 18:34:16 -070090 Switch.__init__(self, name, **kwargs)
Carmelo Cascone3c216fa2018-06-22 14:52:15 +020091 self.grpcPort = grpcport
92 self.thriftPort = thriftport
Carmelo Casconeeaa8b1d2018-04-11 14:12:17 -070093 self.cpuPort = cpuport
Carmelo Casconefb76b042017-07-17 19:42:00 -040094 self.json = json
Carmelo Cascone46d360b2017-08-29 20:20:32 +020095 self.debugger = parseBoolean(debugger)
Carmelo Cascone76e63862018-09-04 14:25:49 -070096 self.notifications = parseBoolean(notifications)
Carmelo Cascone785fada2016-06-16 18:34:16 -070097 self.loglevel = loglevel
Carmelo Casconef11513d2018-01-16 00:31:14 -080098 # Important: Mininet removes all /tmp/*.log files in case of exceptions.
99 # We want to be able to see the bmv2 log if anything goes wrong, hence
100 # avoid the .log extension.
Carmelo Casconec2821332018-05-14 18:15:33 -0700101 self.logfile = '/tmp/bmv2-%s-log' % self.name
Carmelo Cascone46d360b2017-08-29 20:20:32 +0200102 self.elogger = parseBoolean(elogger)
103 self.pktdump = parseBoolean(pktdump)
Carmelo Cascone46d360b2017-08-29 20:20:32 +0200104 self.netcfg = parseBoolean(netcfg)
105 self.dryrun = parseBoolean(dryrun)
106 self.valgrind = parseBoolean(valgrind)
Carmelo Casconec2821332018-05-14 18:15:33 -0700107 self.netcfgfile = '/tmp/bmv2-%s-netcfg.json' % self.name
Carmelo Casconeeaa8b1d2018-04-11 14:12:17 -0700108 self.pipeconfId = pipeconf
109 self.injectPorts = parseBoolean(portcfg)
110 self.withGnmi = parseBoolean(gnmi)
Carmelo Casconef11513d2018-01-16 00:31:14 -0800111 self.longitude = kwargs['longitude'] if 'longitude' in kwargs else None
112 self.latitude = kwargs['latitude'] if 'latitude' in kwargs else None
Carmelo Cascone55965c62018-05-17 18:13:16 -0700113 if onosdevid is not None and len(onosdevid) > 0:
114 self.onosDeviceId = onosdevid
115 else:
116 self.onosDeviceId = "device:bmv2:%s" % self.name
Carmelo Casconef11513d2018-01-16 00:31:14 -0800117 self.logfd = None
118 self.bmv2popen = None
119 self.stopped = False
Yi Tseng7875cb72017-08-08 10:15:58 -0700120
Carmelo Casconef11513d2018-01-16 00:31:14 -0800121 # Remove files from previous executions
122 self.cleanupTmpFiles()
123
Carmelo Casconeb7524272017-06-05 16:53:13 -0400124 def getSourceIp(self, dstIP):
125 """
Carmelo Casconef11513d2018-01-16 00:31:14 -0800126 Queries the Linux routing table to get the source IP that can talk with
127 dstIP, and vice versa.
Carmelo Casconeb7524272017-06-05 16:53:13 -0400128 """
129 ipRouteOut = self.cmd('ip route get %s' % dstIP)
130 r = re.search(r"src (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})", ipRouteOut)
131 return r.group(1) if r else None
132
Yi Tseng7875cb72017-08-08 10:15:58 -0700133 def getDeviceConfig(self, srcIP):
Carmelo Cascone2cad9ef2017-08-01 21:52:07 +0200134
Yi Tseng7875cb72017-08-08 10:15:58 -0700135 basicCfg = {
Carmelo Casconef6cb3e82018-08-24 00:00:28 +0000136 "driver": "bmv2"
Yi Tseng7875cb72017-08-08 10:15:58 -0700137 }
138
139 if self.longitude and self.latitude:
140 basicCfg["longitude"] = self.longitude
141 basicCfg["latitude"] = self.latitude
142
143 cfgData = {
Carmelo Cascone34433252017-08-25 20:27:18 +0200144 "generalprovider": {
145 "p4runtime": {
146 "ip": srcIP,
147 "port": self.grpcPort,
Carmelo Casconec2821332018-05-14 18:15:33 -0700148 "deviceId": BMV2_DEFAULT_DEVICE_ID,
Carmelo Cascone34433252017-08-25 20:27:18 +0200149 "deviceKeyId": "p4runtime:%s" % self.onosDeviceId
Esin Karaman971fb7f2017-12-28 13:44:52 +0000150 },
Carmelo Cascone15693a22018-12-12 19:06:57 -0800151 # "bmv2-thrift": {
152 # "ip": srcIP,
153 # "port": self.thriftPort
154 # }
Carmelo Cascone34433252017-08-25 20:27:18 +0200155 },
156 "piPipeconf": {
157 "piPipeconfId": self.pipeconfId
158 },
Andrea Campanellabf9e5ce2017-12-06 14:26:36 +0100159 "basic": basicCfg
Yi Tseng7875cb72017-08-08 10:15:58 -0700160 }
161
Carmelo Cascone4f985cd2018-02-11 17:36:42 -0800162 if self.withGnmi:
163 cfgData["generalprovider"]["gnmi"] = {
164 "ip": srcIP,
165 "port": self.grpcPort
166 }
167
168 if self.injectPorts:
Andrea Campanellabf9e5ce2017-12-06 14:26:36 +0100169 portData = {}
170 portId = 1
171 for intfName in self.intfNames():
172 if intfName == 'lo':
173 continue
174 portData[str(portId)] = {
175 "number": portId,
176 "name": intfName,
177 "enabled": True,
178 "removed": False,
179 "type": "copper",
180 "speed": 10000
181 }
182 portId += 1
183
184 cfgData['ports'] = portData
185
Yi Tseng7875cb72017-08-08 10:15:58 -0700186 return cfgData
187
188 def doOnosNetcfg(self, controllerIP):
189 """
190 Notifies ONOS about the new device via Netcfg.
191 """
192 srcIP = self.getSourceIp(controllerIP)
193 if not srcIP:
Carmelo Casconef11513d2018-01-16 00:31:14 -0800194 warn("*** WARN: unable to get switch IP address, won't do netcfg\n")
Yi Tseng7875cb72017-08-08 10:15:58 -0700195 return
196
Carmelo Casconea11279b2017-06-22 04:30:08 -0400197 cfgData = {
198 "devices": {
Yi Tseng7875cb72017-08-08 10:15:58 -0700199 self.onosDeviceId: self.getDeviceConfig(srcIP)
Carmelo Casconeb7524272017-06-05 16:53:13 -0400200 }
Carmelo Casconea11279b2017-06-22 04:30:08 -0400201 }
Carmelo Casconeb7524272017-06-05 16:53:13 -0400202 with open(self.netcfgfile, 'w') as fp:
203 json.dump(cfgData, fp, indent=4)
Carmelo Cascone46d360b2017-08-29 20:20:32 +0200204
205 if not self.netcfg:
206 # Do not push config to ONOS.
207 return
208
Brian O'Connor71167f92017-06-16 14:55:00 -0700209 # Build netcfg URL
210 url = 'http://%s:8181/onos/v1/network/configuration/' % controllerIP
211 # Instantiate password manager for HTTP auth
212 pm = urllib2.HTTPPasswordMgrWithDefaultRealm()
Carmelo Casconef11513d2018-01-16 00:31:14 -0800213 pm.add_password(None, url,
214 os.environ['ONOS_WEB_USER'],
215 os.environ['ONOS_WEB_PASS'])
216 urllib2.install_opener(urllib2.build_opener(
217 urllib2.HTTPBasicAuthHandler(pm)))
Brian O'Connor71167f92017-06-16 14:55:00 -0700218 # Push config data to controller
Carmelo Casconef11513d2018-01-16 00:31:14 -0800219 req = urllib2.Request(url, json.dumps(cfgData),
220 {'Content-Type': 'application/json'})
Carmelo Casconea11279b2017-06-22 04:30:08 -0400221 try:
222 f = urllib2.urlopen(req)
223 print f.read()
224 f.close()
225 except urllib2.URLError as e:
Carmelo Casconef11513d2018-01-16 00:31:14 -0800226 warn("*** WARN: unable to push config to ONOS (%s)\n" % e.reason)
Carmelo Casconeb7524272017-06-05 16:53:13 -0400227
Carmelo Cascone785fada2016-06-16 18:34:16 -0700228 def start(self, controllers):
Carmelo Casconef11513d2018-01-16 00:31:14 -0800229 bmv2Args = [SIMPLE_SWITCH_GRPC] + self.grpcTargetArgs()
230 if self.valgrind:
231 bmv2Args = VALGRIND_PREFIX.split() + bmv2Args
232
233 cmdString = " ".join(bmv2Args)
234
235 if self.dryrun:
236 info("\n*** DRY RUN (not executing bmv2)")
237
238 info("\nStarting BMv2 target: %s\n" % cmdString)
239
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200240 writeToFile("/tmp/bmv2-%s-grpc-port" % self.name, self.grpcPort)
241 writeToFile("/tmp/bmv2-%s-thrift-port" % self.name, self.thriftPort)
242
Carmelo Casconef11513d2018-01-16 00:31:14 -0800243 try:
244 if not self.dryrun:
245 # Start the switch
246 self.logfd = open(self.logfile, "w")
247 self.bmv2popen = self.popen(cmdString,
248 stdout=self.logfd,
249 stderr=self.logfd)
250 self.waitBmv2Start()
251 # We want to be notified if BMv2 dies...
252 threading.Thread(target=watchDog, args=[self]).start()
253
254 self.doOnosNetcfg(self.controllerIp(controllers))
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200255 except Exception:
256 ONOSBmv2Switch.mininet_exception = 1
Carmelo Casconef11513d2018-01-16 00:31:14 -0800257 self.killBmv2()
258 self.printBmv2Log()
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200259 raise
Carmelo Casconef11513d2018-01-16 00:31:14 -0800260
261 def grpcTargetArgs(self):
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200262 if self.grpcPort is None:
263 self.grpcPort = pickUnusedPort()
264 if self.thriftPort is None:
265 self.thriftPort = pickUnusedPort()
Carmelo Casconec2821332018-05-14 18:15:33 -0700266 args = ['--device-id %s' % str(BMV2_DEFAULT_DEVICE_ID)]
Carmelo Cascone785fada2016-06-16 18:34:16 -0700267 for port, intf in self.intfs.items():
268 if not intf.IP():
269 args.append('-i %d@%s' % (port, intf.name))
Carmelo Casconec2821332018-05-14 18:15:33 -0700270 args.append('--thrift-port %s' % self.thriftPort)
Carmelo Cascone76e63862018-09-04 14:25:49 -0700271 if self.notifications:
272 ntfaddr = 'ipc:///tmp/bmv2-%s-notifications.ipc' % self.name
273 args.append('--notifications-addr %s' % ntfaddr)
Carmelo Cascone785fada2016-06-16 18:34:16 -0700274 if self.elogger:
Carmelo Casconec2821332018-05-14 18:15:33 -0700275 nanologaddr = 'ipc:///tmp/bmv2-%s-nanolog.ipc' % self.name
276 args.append('--nanolog %s' % nanologaddr)
Carmelo Cascone785fada2016-06-16 18:34:16 -0700277 if self.debugger:
Carmelo Casconec2821332018-05-14 18:15:33 -0700278 dbgaddr = 'ipc:///tmp/bmv2-%s-debug.ipc' % self.name
279 args.append('--debugger-addr %s' % dbgaddr)
Carmelo Casconefb76b042017-07-17 19:42:00 -0400280 args.append('--log-console')
Carmelo Cascone34433252017-08-25 20:27:18 +0200281 if self.pktdump:
Carmelo Casconef11513d2018-01-16 00:31:14 -0800282 args.append('--pcap --dump-packet-data %s' % PKT_BYTES_TO_DUMP)
Carmelo Cascone9e6621f2017-06-27 16:06:33 -0400283 args.append('-L%s' % self.loglevel)
Carmelo Casconefb76b042017-07-17 19:42:00 -0400284 if not self.json:
285 args.append('--no-p4')
286 else:
287 args.append(self.json)
Carmelo Casconef11513d2018-01-16 00:31:14 -0800288 # gRPC target-specific options
Carmelo Cascone785fada2016-06-16 18:34:16 -0700289 args.append('--')
Carmelo Casconef11513d2018-01-16 00:31:14 -0800290 args.append('--cpu-port %s' % self.cpuPort)
291 args.append('--grpc-server-addr 0.0.0.0:%s' % self.grpcPort)
292 return args
Carmelo Cascone785fada2016-06-16 18:34:16 -0700293
Carmelo Casconef11513d2018-01-16 00:31:14 -0800294 def waitBmv2Start(self):
295 # Wait for switch to open gRPC port, before sending ONOS the netcfg.
296 # Include time-out just in case something hangs.
297 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
298 endtime = time.time() + SWITCH_START_TIMEOUT
299 while True:
300 result = sock.connect_ex(('127.0.0.1', self.grpcPort))
301 if result == 0:
302 # The port is open. Let's go! (Close socket first)
303 sock.close()
304 break
305 # Port is not open yet. If there is time, we wait a bit.
306 if endtime > time.time():
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200307 time.sleep(0.1)
Carmelo Casconef11513d2018-01-16 00:31:14 -0800308 else:
309 # Time's up.
310 raise Exception("Switch did not start before timeout")
Carmelo Casconefb76b042017-07-17 19:42:00 -0400311
Carmelo Casconef11513d2018-01-16 00:31:14 -0800312 def printBmv2Log(self):
313 if os.path.isfile(self.logfile):
314 print "-" * 80
Carmelo Casconec2821332018-05-14 18:15:33 -0700315 print "%s log (from %s):" % (self.name, self.logfile)
Carmelo Casconef11513d2018-01-16 00:31:14 -0800316 with open(self.logfile, 'r') as f:
317 lines = f.readlines()
318 if len(lines) > BMV2_LOG_LINES:
319 print "..."
320 for line in lines[-BMV2_LOG_LINES:]:
321 print line.rstrip()
Carmelo Casconefb76b042017-07-17 19:42:00 -0400322
Carmelo Casconef11513d2018-01-16 00:31:14 -0800323 @staticmethod
324 def controllerIp(controllers):
325 try:
326 # onos.py
Carmelo Cascone46d360b2017-08-29 20:20:32 +0200327 clist = controllers[0].nodes()
328 except AttributeError:
329 clist = controllers
330 assert len(clist) > 0
Carmelo Cascone44448a52018-06-25 23:36:57 +0200331 return random.choice(clist).IP()
Keesjan Karsten8539f082018-01-04 17:03:31 +0100332
Carmelo Casconef11513d2018-01-16 00:31:14 -0800333 def killBmv2(self, log=False):
334 if self.bmv2popen is not None:
335 self.bmv2popen.kill()
336 if self.logfd is not None:
337 if log:
338 self.logfd.write("*** PROCESS TERMINATED BY MININET ***\n")
339 self.logfd.close()
Keesjan Karsten8539f082018-01-04 17:03:31 +0100340
Carmelo Casconef11513d2018-01-16 00:31:14 -0800341 def cleanupTmpFiles(self):
Carmelo Casconec2821332018-05-14 18:15:33 -0700342 self.cmd("rm -f /tmp/bmv2-%s-*" % self.name)
Carmelo Casconeb7524272017-06-05 16:53:13 -0400343
344 def stop(self, deleteIntfs=True):
345 """Terminate switch."""
Carmelo Casconef11513d2018-01-16 00:31:14 -0800346 self.stopped = True
347 self.killBmv2(log=True)
Carmelo Casconeb7524272017-06-05 16:53:13 -0400348 Switch.stop(self, deleteIntfs)
Carmelo Cascone785fada2016-06-16 18:34:16 -0700349
350
Carmelo Casconeb7524272017-06-05 16:53:13 -0400351# Exports for bin/mn
Carmelo Cascone785fada2016-06-16 18:34:16 -0700352switches = {'onosbmv2': ONOSBmv2Switch}
Carmelo Cascone34433252017-08-25 20:27:18 +0200353hosts = {'onoshost': ONOSHost}