blob: 0c0fc1502d5d8949571ff1a64e5a629c705cb62d [file] [log] [blame]
Carmelo Cascone44448a52018-06-25 23:36:57 +02001import json
Carmelo Cascone3c216fa2018-06-22 14:52:15 +02002import multiprocessing
Carmelo Casconeb7524272017-06-05 16:53:13 -04003import os
Carmelo Cascone44448a52018-06-25 23:36:57 +02004import random
Carmelo Casconeb7524272017-06-05 16:53:13 -04005import re
Carmelo Cascone44448a52018-06-25 23:36:57 +02006import socket
Carmelo Casconef11513d2018-01-16 00:31:14 -08007import threading
Carmelo Cascone46d360b2017-08-29 20:20:32 +02008import time
Carmelo Cascone44448a52018-06-25 23:36:57 +02009import urllib2
Carmelo Casconef11513d2018-01-16 00:31:14 -080010from contextlib import closing
Carmelo Cascone44448a52018-06-25 23:36:57 +020011from mininet.log import info, warn
Carmelo Cascone34433252017-08-25 20:27:18 +020012from mininet.node import Switch, Host
Carmelo Cascone785fada2016-06-16 18:34:16 -070013
Carmelo Casconef11513d2018-01-16 00:31:14 -080014SIMPLE_SWITCH_GRPC = 'simple_switch_grpc'
Carmelo Cascone34433252017-08-25 20:27:18 +020015PKT_BYTES_TO_DUMP = 80
Carmelo Cascone46d360b2017-08-29 20:20:32 +020016VALGRIND_PREFIX = 'valgrind --leak-check=yes'
Carmelo Casconef11513d2018-01-16 00:31:14 -080017SWITCH_START_TIMEOUT = 5 # seconds
18BMV2_LOG_LINES = 5
Carmelo Casconec2821332018-05-14 18:15:33 -070019BMV2_DEFAULT_DEVICE_ID = 0
Carmelo Casconef11513d2018-01-16 00:31:14 -080020
Carmelo Cascone46d360b2017-08-29 20:20:32 +020021
22def parseBoolean(value):
23 if value in ['1', 1, 'true', 'True']:
24 return True
25 else:
26 return False
Carmelo Cascone34433252017-08-25 20:27:18 +020027
28
Carmelo Casconef11513d2018-01-16 00:31:14 -080029def pickUnusedPort():
30 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
31 s.bind(('localhost', 0))
32 addr, port = s.getsockname()
33 s.close()
34 return port
35
36
37def writeToFile(path, value):
38 with open(path, "w") as f:
39 f.write(str(value))
40
41
42def watchDog(sw):
43 while True:
Carmelo Cascone3c216fa2018-06-22 14:52:15 +020044 if ONOSBmv2Switch.mininet_exception == 1:
45 sw.killBmv2(log=False)
46 return
Carmelo Casconef11513d2018-01-16 00:31:14 -080047 if sw.stopped:
48 return
49 with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
50 if s.connect_ex(('127.0.0.1', sw.grpcPort)) == 0:
51 time.sleep(1)
52 else:
Carmelo Casconedf32bea2018-06-19 17:45:54 +020053 warn("\n*** WARN: BMv2 instance %s died!\n" % sw.name)
Carmelo Casconef11513d2018-01-16 00:31:14 -080054 sw.printBmv2Log()
55 print ("-" * 80) + "\n"
56 return
57
58
Carmelo Cascone34433252017-08-25 20:27:18 +020059class ONOSHost(Host):
60 def __init__(self, name, inNamespace=True, **params):
61 Host.__init__(self, name, inNamespace=inNamespace, **params)
62
63 def config(self, **params):
64 r = super(Host, self).config(**params)
65 for off in ["rx", "tx", "sg"]:
Carmelo Casconef11513d2018-01-16 00:31:14 -080066 cmd = "/sbin/ethtool --offload %s %s off" \
Carmelo Cascone6ec8f8f2017-11-22 14:27:06 -080067 % (self.defaultIntf(), off)
Carmelo Cascone34433252017-08-25 20:27:18 +020068 self.cmd(cmd)
69 # disable IPv6
70 self.cmd("sysctl -w net.ipv6.conf.all.disable_ipv6=1")
71 self.cmd("sysctl -w net.ipv6.conf.default.disable_ipv6=1")
72 self.cmd("sysctl -w net.ipv6.conf.lo.disable_ipv6=1")
73 return r
Carmelo Casconeb7524272017-06-05 16:53:13 -040074
Carmelo Cascone785fada2016-06-16 18:34:16 -070075
76class ONOSBmv2Switch(Switch):
Carmelo Casconeb7524272017-06-05 16:53:13 -040077 """BMv2 software switch with gRPC server"""
Carmelo Cascone3c216fa2018-06-22 14:52:15 +020078 # Shared value used to notify to all instances of this class that a Mininet
79 # exception occurred. Mininet exception handling doesn't call the stop()
80 # method, so the mn process would hang after clean-up since Bmv2 would still
81 # be running.
82 mininet_exception = multiprocessing.Value('i', 0)
Carmelo Cascone785fada2016-06-16 18:34:16 -070083
Carmelo Casconef11513d2018-01-16 00:31:14 -080084 def __init__(self, name, json=None, debugger=False, loglevel="warn",
Carmelo Casconeeaa8b1d2018-04-11 14:12:17 -070085 elogger=False, grpcport=None, cpuport=255,
86 thriftport=None, netcfg=True, dryrun=False, pipeconf="",
87 pktdump=False, valgrind=False, gnmi=False,
Carmelo Cascone55965c62018-05-17 18:13:16 -070088 portcfg=True, onosdevid=None, **kwargs):
Carmelo Cascone785fada2016-06-16 18:34:16 -070089 Switch.__init__(self, name, **kwargs)
Carmelo Cascone3c216fa2018-06-22 14:52:15 +020090 self.grpcPort = grpcport
91 self.thriftPort = thriftport
Carmelo Casconeeaa8b1d2018-04-11 14:12:17 -070092 self.cpuPort = cpuport
Carmelo Casconefb76b042017-07-17 19:42:00 -040093 self.json = json
Carmelo Cascone46d360b2017-08-29 20:20:32 +020094 self.debugger = parseBoolean(debugger)
Carmelo Cascone785fada2016-06-16 18:34:16 -070095 self.loglevel = loglevel
Carmelo Casconef11513d2018-01-16 00:31:14 -080096 # Important: Mininet removes all /tmp/*.log files in case of exceptions.
97 # We want to be able to see the bmv2 log if anything goes wrong, hence
98 # avoid the .log extension.
Carmelo Casconec2821332018-05-14 18:15:33 -070099 self.logfile = '/tmp/bmv2-%s-log' % self.name
Carmelo Cascone46d360b2017-08-29 20:20:32 +0200100 self.elogger = parseBoolean(elogger)
101 self.pktdump = parseBoolean(pktdump)
Carmelo Cascone46d360b2017-08-29 20:20:32 +0200102 self.netcfg = parseBoolean(netcfg)
103 self.dryrun = parseBoolean(dryrun)
104 self.valgrind = parseBoolean(valgrind)
Carmelo Casconec2821332018-05-14 18:15:33 -0700105 self.netcfgfile = '/tmp/bmv2-%s-netcfg.json' % self.name
Carmelo Casconeeaa8b1d2018-04-11 14:12:17 -0700106 self.pipeconfId = pipeconf
107 self.injectPorts = parseBoolean(portcfg)
108 self.withGnmi = parseBoolean(gnmi)
Carmelo Casconef11513d2018-01-16 00:31:14 -0800109 self.longitude = kwargs['longitude'] if 'longitude' in kwargs else None
110 self.latitude = kwargs['latitude'] if 'latitude' in kwargs else None
Carmelo Cascone55965c62018-05-17 18:13:16 -0700111 if onosdevid is not None and len(onosdevid) > 0:
112 self.onosDeviceId = onosdevid
113 else:
114 self.onosDeviceId = "device:bmv2:%s" % self.name
Carmelo Casconef11513d2018-01-16 00:31:14 -0800115 self.logfd = None
116 self.bmv2popen = None
117 self.stopped = False
Yi Tseng7875cb72017-08-08 10:15:58 -0700118
Carmelo Casconef11513d2018-01-16 00:31:14 -0800119 # Remove files from previous executions
120 self.cleanupTmpFiles()
121
Carmelo Casconeb7524272017-06-05 16:53:13 -0400122 def getSourceIp(self, dstIP):
123 """
Carmelo Casconef11513d2018-01-16 00:31:14 -0800124 Queries the Linux routing table to get the source IP that can talk with
125 dstIP, and vice versa.
Carmelo Casconeb7524272017-06-05 16:53:13 -0400126 """
127 ipRouteOut = self.cmd('ip route get %s' % dstIP)
128 r = re.search(r"src (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})", ipRouteOut)
129 return r.group(1) if r else None
130
Yi Tseng7875cb72017-08-08 10:15:58 -0700131 def getDeviceConfig(self, srcIP):
Carmelo Cascone2cad9ef2017-08-01 21:52:07 +0200132
Yi Tseng7875cb72017-08-08 10:15:58 -0700133 basicCfg = {
Carmelo Casconedccbdf52018-08-24 00:00:28 +0000134 "driver": "bmv2"
Yi Tseng7875cb72017-08-08 10:15:58 -0700135 }
136
137 if self.longitude and self.latitude:
138 basicCfg["longitude"] = self.longitude
139 basicCfg["latitude"] = self.latitude
140
141 cfgData = {
Carmelo Cascone34433252017-08-25 20:27:18 +0200142 "generalprovider": {
143 "p4runtime": {
144 "ip": srcIP,
145 "port": self.grpcPort,
Carmelo Casconec2821332018-05-14 18:15:33 -0700146 "deviceId": BMV2_DEFAULT_DEVICE_ID,
Carmelo Cascone34433252017-08-25 20:27:18 +0200147 "deviceKeyId": "p4runtime:%s" % self.onosDeviceId
Esin Karaman971fb7f2017-12-28 13:44:52 +0000148 },
149 "bmv2-thrift": {
150 "ip": srcIP,
151 "port": self.thriftPort
Carmelo Cascone34433252017-08-25 20:27:18 +0200152 }
153 },
154 "piPipeconf": {
155 "piPipeconfId": self.pipeconfId
156 },
Andrea Campanellabf9e5ce2017-12-06 14:26:36 +0100157 "basic": basicCfg
Yi Tseng7875cb72017-08-08 10:15:58 -0700158 }
159
Carmelo Cascone4f985cd2018-02-11 17:36:42 -0800160 if self.withGnmi:
161 cfgData["generalprovider"]["gnmi"] = {
162 "ip": srcIP,
163 "port": self.grpcPort
164 }
165
166 if self.injectPorts:
Andrea Campanellabf9e5ce2017-12-06 14:26:36 +0100167 portData = {}
168 portId = 1
169 for intfName in self.intfNames():
170 if intfName == 'lo':
171 continue
172 portData[str(portId)] = {
173 "number": portId,
174 "name": intfName,
175 "enabled": True,
176 "removed": False,
177 "type": "copper",
178 "speed": 10000
179 }
180 portId += 1
181
182 cfgData['ports'] = portData
183
Yi Tseng7875cb72017-08-08 10:15:58 -0700184 return cfgData
185
186 def doOnosNetcfg(self, controllerIP):
187 """
188 Notifies ONOS about the new device via Netcfg.
189 """
190 srcIP = self.getSourceIp(controllerIP)
191 if not srcIP:
Carmelo Casconef11513d2018-01-16 00:31:14 -0800192 warn("*** WARN: unable to get switch IP address, won't do netcfg\n")
Yi Tseng7875cb72017-08-08 10:15:58 -0700193 return
194
Carmelo Casconea11279b2017-06-22 04:30:08 -0400195 cfgData = {
196 "devices": {
Yi Tseng7875cb72017-08-08 10:15:58 -0700197 self.onosDeviceId: self.getDeviceConfig(srcIP)
Carmelo Casconeb7524272017-06-05 16:53:13 -0400198 }
Carmelo Casconea11279b2017-06-22 04:30:08 -0400199 }
Carmelo Casconeb7524272017-06-05 16:53:13 -0400200 with open(self.netcfgfile, 'w') as fp:
201 json.dump(cfgData, fp, indent=4)
Carmelo Cascone46d360b2017-08-29 20:20:32 +0200202
203 if not self.netcfg:
204 # Do not push config to ONOS.
205 return
206
Brian O'Connor71167f92017-06-16 14:55:00 -0700207 # Build netcfg URL
208 url = 'http://%s:8181/onos/v1/network/configuration/' % controllerIP
209 # Instantiate password manager for HTTP auth
210 pm = urllib2.HTTPPasswordMgrWithDefaultRealm()
Carmelo Casconef11513d2018-01-16 00:31:14 -0800211 pm.add_password(None, url,
212 os.environ['ONOS_WEB_USER'],
213 os.environ['ONOS_WEB_PASS'])
214 urllib2.install_opener(urllib2.build_opener(
215 urllib2.HTTPBasicAuthHandler(pm)))
Brian O'Connor71167f92017-06-16 14:55:00 -0700216 # Push config data to controller
Carmelo Casconef11513d2018-01-16 00:31:14 -0800217 req = urllib2.Request(url, json.dumps(cfgData),
218 {'Content-Type': 'application/json'})
Carmelo Casconea11279b2017-06-22 04:30:08 -0400219 try:
220 f = urllib2.urlopen(req)
221 print f.read()
222 f.close()
223 except urllib2.URLError as e:
Carmelo Casconef11513d2018-01-16 00:31:14 -0800224 warn("*** WARN: unable to push config to ONOS (%s)\n" % e.reason)
Carmelo Casconeb7524272017-06-05 16:53:13 -0400225
Carmelo Cascone785fada2016-06-16 18:34:16 -0700226 def start(self, controllers):
Carmelo Casconef11513d2018-01-16 00:31:14 -0800227 bmv2Args = [SIMPLE_SWITCH_GRPC] + self.grpcTargetArgs()
228 if self.valgrind:
229 bmv2Args = VALGRIND_PREFIX.split() + bmv2Args
230
231 cmdString = " ".join(bmv2Args)
232
233 if self.dryrun:
234 info("\n*** DRY RUN (not executing bmv2)")
235
236 info("\nStarting BMv2 target: %s\n" % cmdString)
237
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200238 writeToFile("/tmp/bmv2-%s-grpc-port" % self.name, self.grpcPort)
239 writeToFile("/tmp/bmv2-%s-thrift-port" % self.name, self.thriftPort)
240
Carmelo Casconef11513d2018-01-16 00:31:14 -0800241 try:
242 if not self.dryrun:
243 # Start the switch
244 self.logfd = open(self.logfile, "w")
245 self.bmv2popen = self.popen(cmdString,
246 stdout=self.logfd,
247 stderr=self.logfd)
248 self.waitBmv2Start()
249 # We want to be notified if BMv2 dies...
250 threading.Thread(target=watchDog, args=[self]).start()
251
252 self.doOnosNetcfg(self.controllerIp(controllers))
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200253 except Exception:
254 ONOSBmv2Switch.mininet_exception = 1
Carmelo Casconef11513d2018-01-16 00:31:14 -0800255 self.killBmv2()
256 self.printBmv2Log()
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200257 raise
Carmelo Casconef11513d2018-01-16 00:31:14 -0800258
259 def grpcTargetArgs(self):
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200260 if self.grpcPort is None:
261 self.grpcPort = pickUnusedPort()
262 if self.thriftPort is None:
263 self.thriftPort = pickUnusedPort()
Carmelo Casconec2821332018-05-14 18:15:33 -0700264 args = ['--device-id %s' % str(BMV2_DEFAULT_DEVICE_ID)]
Carmelo Cascone785fada2016-06-16 18:34:16 -0700265 for port, intf in self.intfs.items():
266 if not intf.IP():
267 args.append('-i %d@%s' % (port, intf.name))
Carmelo Casconec2821332018-05-14 18:15:33 -0700268 args.append('--thrift-port %s' % self.thriftPort)
269 ntfaddr = 'ipc:///tmp/bmv2-%s-notifications.ipc' % self.name
270 args.append('--notifications-addr %s' % ntfaddr)
Carmelo Cascone785fada2016-06-16 18:34:16 -0700271 if self.elogger:
Carmelo Casconec2821332018-05-14 18:15:33 -0700272 nanologaddr = 'ipc:///tmp/bmv2-%s-nanolog.ipc' % self.name
273 args.append('--nanolog %s' % nanologaddr)
Carmelo Cascone785fada2016-06-16 18:34:16 -0700274 if self.debugger:
Carmelo Casconec2821332018-05-14 18:15:33 -0700275 dbgaddr = 'ipc:///tmp/bmv2-%s-debug.ipc' % self.name
276 args.append('--debugger-addr %s' % dbgaddr)
Carmelo Casconefb76b042017-07-17 19:42:00 -0400277 args.append('--log-console')
Carmelo Cascone34433252017-08-25 20:27:18 +0200278 if self.pktdump:
Carmelo Casconef11513d2018-01-16 00:31:14 -0800279 args.append('--pcap --dump-packet-data %s' % PKT_BYTES_TO_DUMP)
Carmelo Cascone9e6621f2017-06-27 16:06:33 -0400280 args.append('-L%s' % self.loglevel)
Carmelo Casconefb76b042017-07-17 19:42:00 -0400281 if not self.json:
282 args.append('--no-p4')
283 else:
284 args.append(self.json)
Carmelo Casconef11513d2018-01-16 00:31:14 -0800285 # gRPC target-specific options
Carmelo Cascone785fada2016-06-16 18:34:16 -0700286 args.append('--')
Carmelo Casconef11513d2018-01-16 00:31:14 -0800287 args.append('--cpu-port %s' % self.cpuPort)
288 args.append('--grpc-server-addr 0.0.0.0:%s' % self.grpcPort)
289 return args
Carmelo Cascone785fada2016-06-16 18:34:16 -0700290
Carmelo Casconef11513d2018-01-16 00:31:14 -0800291 def waitBmv2Start(self):
292 # Wait for switch to open gRPC port, before sending ONOS the netcfg.
293 # Include time-out just in case something hangs.
294 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
295 endtime = time.time() + SWITCH_START_TIMEOUT
296 while True:
297 result = sock.connect_ex(('127.0.0.1', self.grpcPort))
298 if result == 0:
299 # The port is open. Let's go! (Close socket first)
300 sock.close()
301 break
302 # Port is not open yet. If there is time, we wait a bit.
303 if endtime > time.time():
Carmelo Cascone3c216fa2018-06-22 14:52:15 +0200304 time.sleep(0.1)
Carmelo Casconef11513d2018-01-16 00:31:14 -0800305 else:
306 # Time's up.
307 raise Exception("Switch did not start before timeout")
Carmelo Casconefb76b042017-07-17 19:42:00 -0400308
Carmelo Casconef11513d2018-01-16 00:31:14 -0800309 def printBmv2Log(self):
310 if os.path.isfile(self.logfile):
311 print "-" * 80
Carmelo Casconec2821332018-05-14 18:15:33 -0700312 print "%s log (from %s):" % (self.name, self.logfile)
Carmelo Casconef11513d2018-01-16 00:31:14 -0800313 with open(self.logfile, 'r') as f:
314 lines = f.readlines()
315 if len(lines) > BMV2_LOG_LINES:
316 print "..."
317 for line in lines[-BMV2_LOG_LINES:]:
318 print line.rstrip()
Carmelo Casconefb76b042017-07-17 19:42:00 -0400319
Carmelo Casconef11513d2018-01-16 00:31:14 -0800320 @staticmethod
321 def controllerIp(controllers):
322 try:
323 # onos.py
Carmelo Cascone46d360b2017-08-29 20:20:32 +0200324 clist = controllers[0].nodes()
325 except AttributeError:
326 clist = controllers
327 assert len(clist) > 0
Carmelo Cascone44448a52018-06-25 23:36:57 +0200328 return random.choice(clist).IP()
Keesjan Karsten8539f082018-01-04 17:03:31 +0100329
Carmelo Casconef11513d2018-01-16 00:31:14 -0800330 def killBmv2(self, log=False):
331 if self.bmv2popen is not None:
332 self.bmv2popen.kill()
333 if self.logfd is not None:
334 if log:
335 self.logfd.write("*** PROCESS TERMINATED BY MININET ***\n")
336 self.logfd.close()
Keesjan Karsten8539f082018-01-04 17:03:31 +0100337
Carmelo Casconef11513d2018-01-16 00:31:14 -0800338 def cleanupTmpFiles(self):
Carmelo Casconec2821332018-05-14 18:15:33 -0700339 self.cmd("rm -f /tmp/bmv2-%s-*" % self.name)
Carmelo Casconeb7524272017-06-05 16:53:13 -0400340
341 def stop(self, deleteIntfs=True):
342 """Terminate switch."""
Carmelo Casconef11513d2018-01-16 00:31:14 -0800343 self.stopped = True
344 self.killBmv2(log=True)
Carmelo Casconeb7524272017-06-05 16:53:13 -0400345 Switch.stop(self, deleteIntfs)
Carmelo Cascone785fada2016-06-16 18:34:16 -0700346
347
Carmelo Casconeb7524272017-06-05 16:53:13 -0400348# Exports for bin/mn
Carmelo Cascone785fada2016-06-16 18:34:16 -0700349switches = {'onosbmv2': ONOSBmv2Switch}
Carmelo Cascone34433252017-08-25 20:27:18 +0200350hosts = {'onoshost': ONOSHost}