blob: 3829250148657e11dba78e0296229ce9f3b4133a [file] [log] [blame]
Jordan Halterman4d32a582017-05-26 19:14:30 -07001#!/bin/bash
2# ---------------------------------
3# Disrupts links between ONOS nodes
4# ---------------------------------
5
6[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
7. $ONOS_ROOT/tools/build/envDefaults
8
9function print_usage {
10 command_name=`basename $0`
11 echo "Disrupts links between ONOS nodes."
12 echo
13 echo "Usage: $command_name [<HOST>] <COMMAND> <ARGS>"
14 echo
15 echo "Options:"
16 echo " HOST The host on which to run the command"
17 echo " COMMAND The command to run"
18 echo " ARGS The command arguments"
19 echo " [-h | --help] Print this help"
20 echo
Jordan Halterman862d4f92018-01-25 17:19:30 -080021 echo "COMMAND: <crash | partition | isolate | partition-halves | partition-bridge | heal | delay | drop | reorder | duplicate | corrupt | restore>"
Jordan Halterman4d32a582017-05-26 19:14:30 -070022 echo
23 echo " crash Crashes the given host using \"kill -9\""
24 echo
25 echo " Syntax:"
26 echo " $command_name <HOST> crash"
27 echo
28 echo " Examples:"
29 echo " $command_name 10.127.10.111 crash"
30 echo
Jordan Haltermanddc2b182017-07-23 01:27:47 -070031 echo " recover Recovers the given host"
32 echo
33 echo " Syntax:"
34 echo " $command_name <host> recover"
35 echo
36 echo " Examples:"
37 echo " $command_name 10.127.10.111 recover"
38 echo
Jordan Halterman4d32a582017-05-26 19:14:30 -070039 echo " partition Partitions the given host from a list of source host or all ONOS hosts if no sources are specified"
40 echo
41 echo " Syntax:"
42 echo " $command_name <HOST> partition [<SOURCE> [<SOURCE>...]]"
43 echo
44 echo " Examples:"
45 echo " $command_name 10.127.10.111 partition"
46 echo " $command_name 10.127.10.111 partition 10.127.10.111"
47 echo " $command_name 10.127.10.111 partition 10.127.10.112 10.127.10.113"
48 echo
Jordan Halterman862d4f92018-01-25 17:19:30 -080049 echo " isolate Isolate the given host from all other hosts"
50 echo
51 echo " Syntax:"
52 echo " $command_name <HOST> isolate"
53 echo
54 echo " Examples:"
55 echo " $command_name 10.127.10.111 isolate"
56 echo
Jordan Halterman4d32a582017-05-26 19:14:30 -070057 echo " partition-halves Splits the cluster into two partitions - this is only recommended for clusters with an odd number of nodes"
58 echo
59 echo " Syntax:"
60 echo " $command_name partition-halves"
61 echo
62 echo " Examples:"
63 echo " $command_name partition-halves"
64 echo
65 echo " partition-bridge Creates a bridge partition where the cluster is partitioned into two halves and the target host is connected to both halves"
66 echo
67 echo " Syntax:"
Jordan Haltermanddc2b182017-07-23 01:27:47 -070068 echo " $command_name <HOST> partition-bridge"
Jordan Halterman4d32a582017-05-26 19:14:30 -070069 echo
70 echo " Examples:"
Jordan Haltermanddc2b182017-07-23 01:27:47 -070071 echo " $command_name 10.127.10.111 partition-bridge"
Jordan Halterman4d32a582017-05-26 19:14:30 -070072 echo
73 echo " heal Heals a partition on the given host"
74 echo
75 echo " Syntax:"
76 echo " $command_name [<HOST>] heal"
77 echo
78 echo " Examples:"
79 echo " $command_name heal"
80 echo " $command_name 10.127.10.111 heal"
81 echo
82 echo " delay Delays packets incoming to the given host by an optional LATENCY, JITTER, CORRELATION, and DISTRIBUTION"
83 echo
84 echo " Syntax:"
85 echo " $command_name [<HOST>] delay [<LATENCY> [<JITTER> [<CORRELATION> [<DISTRIBUTION>]]]]"
86 echo
87 echo " Examples:"
88 echo " $command_name delay"
89 echo " $command_name delay 50ms"
90 echo " $command_name delay 50ms 10ms"
91 echo " $command_name delay 50ms 10ms 75%"
92 echo " $command_name delay 50ms 10ms 75% normal"
93 echo " $command_name 10.127.10.111 delay"
94 echo " $command_name 10.127.10.111 delay 50ms"
95 echo " $command_name 10.127.10.111 delay 50ms 10ms"
96 echo " $command_name 10.127.10.111 delay 50ms 10ms 75%"
97 echo " $command_name 10.127.10.111 delay 50ms 10ms 75% normal"
98 echo
99 echo " drop Drops packets incoming to the given host by an optional PROBABILITY and CORRELATION"
100 echo
101 echo " Syntax:"
102 echo " $command_name [<HOST>] drop [<PROBABILITY> [<CORRELATION>]]"
103 echo
104 echo " Examples:"
105 echo " $command_name drop"
106 echo " $command_name drop 2%"
107 echo " $command_name drop 2% 25%"
108 echo " $command_name 10.127.10.111 drop"
109 echo " $command_name 10.127.10.111 drop 2%"
110 echo " $command_name 10.127.10.111 drop 2% 25%"
111 echo
112 echo " reorder Reorders packets incoming to the given host by an optional PROBABILITY and CORRELATION"
113 echo
114 echo " Syntax:"
115 echo " $command_name [<HOST>] reorder [<PROBABILITY> [<CORRELATION>]]"
116 echo
117 echo " Examples:"
118 echo " $command_name reorder"
119 echo " $command_name reorder 2%"
120 echo " $command_name reorder 2% 50%"
121 echo " $command_name 10.127.10.111 reorder"
122 echo " $command_name 10.127.10.111 reorder 2%"
123 echo " $command_name 10.127.10.111 reorder 2% 50%"
124 echo
125 echo " duplicate Duplicates packets incoming to the given host by an optional PROBABILITY and CORRELATION"
126 echo
127 echo " Syntax:"
128 echo " $command_name [<HOST>] duplicate [<PROBABILITY> [<CORRELATION>]]"
129 echo
130 echo " Examples:"
131 echo " $command_name duplicate"
132 echo " $command_name duplicate 0.5%"
133 echo " $command_name duplicate 0.5% 5%"
134 echo " $command_name 10.127.10.111 duplicate"
135 echo " $command_name 10.127.10.111 duplicate 0.5%"
136 echo " $command_name 10.127.10.111 duplicate 0.5% 5%"
137 echo
138 echo " corrupt Corrupts packets incoming to the given host by an optional PROBABILITY"
139 echo
140 echo " Syntax:"
141 echo " $command_name [<HOST>] corrupt [<PROBABILITY>]"
142 echo
143 echo " Examples:"
144 echo " $command_name corrupt"
145 echo " $command_name corrupt 0.1%"
146 echo " $command_name 10.127.10.111 corrupt"
147 echo " $command_name 10.127.10.111 corrupt 0.1%"
148 echo
149 echo " restore Restores the given host from all delay/drop/reorder/duplicate/corrupt states"
150 echo
151 echo " Syntax:"
152 echo " $command_name [<HOST>] restore"
153 echo
154 echo " Examples:"
155 echo " $command_name restore"
156 echo " $command_name 10.127.10.111 restore"
157 echo
158}
159
160# Print usage
161if [ "${1}" = "-h" -o "${1}" = "--help" ]; then
162 print_usage
163 exit 0
164fi
165
166# Converts an argument into a milliseconds string unless it's already in the correct format.
167function to_millis() {
168 if [[ ${1: -2} != "ms" ]]; then
169 echo "${1}ms"
170 else
171 echo ${1}
172 fi
173}
174
175function onos_nodes() {
176 echo $(env | sort | egrep "^OC[0-9]+" | cut -d= -f2)
177}
178
179function partition() {
180 local=${1}
181 remote=${2}
182 echo "Severing link ${remote}->${local}"
183 ssh $ONOS_USER@${local} "sudo iptables -A INPUT -s ${remote} -j DROP -w"
184}
185
186function heal() {
187 local=${1}
188 remote=${2}
189 echo "Restoring link ${remote}->${local}"
190 ssh $ONOS_USER@${local} "sudo iptables -D INPUT -s ${remote} -j DROP -w" &>/dev/null
191}
192
193function delay() {
194 host=${1}
195 latency=$(to_millis ${2:-"50ms"})
196 jitter=$(to_millis ${3:-"10ms"})
197 correlation=${4:-"75%"}
198 distribution=${5:-"normal"}
199 echo "Delaying packets for ${host} (latency: ${latency}, jitter: ${jitter}, correlation: ${correlation}, distribution: ${distribution})"
200 ssh $ONOS_USER@${host} "sudo tc qdisc add dev eth0 root netem delay ${latency} ${jitter} ${correlation} distribution ${distribution}"
201}
202
203function drop() {
204 host=${1}
205 probability=${2:-"2%"}
206 correlation=${3:-"25%"}
207 echo "Dropping packets for ${host} (probability: ${probability}, correlation: ${correlation})"
208 ssh $ONOS_USER@${host} "sudo tc qdisc add dev eth0 root netem loss ${probability} ${correlation}"
209}
210
211function reorder() {
212 host=${1}
213 probability=${2:-"2%"}
214 correlation=${3:-"50%"}
215 echo "Reordering packets for ${host} (probability: ${probability}, correlation: ${correlation})"
216 ssh $ONOS_USER@${host} "sudo tc qdisc add dev eth0 root netem reorder ${probability} ${correlation}"
217}
218
219function duplicate() {
220 host=${1}
221 probability=${2:-"0.5%"}
222 correlation=${3:-"5%"}
223 echo "Duplicating packets for ${host} (probability: ${probability}, correlation: ${correlation})"
224 ssh $ONOS_USER@${host} "sudo tc qdisc add dev eth0 root netem duplicate ${probability} ${correlation}"
225}
226
227function corrupt() {
228 host=${1}
229 probability=${2:-"0.1%"}
230 echo "Corrupting packets for ${host} (probability: ${probability})"
231 ssh $ONOS_USER@${host} "sudo tc qdisc add dev eth0 root netem corrupt ${probability}"
232}
233
234function restore() {
235 host=${1}
236 echo "Restoring packets for ${host}"
237 ssh $ONOS_USER@${host} "sudo tc qdisc del dev eth0 root" &>/dev/null
238}
239
240cmd=${1} && shift
241
242case $cmd in
243
244# Partitions the cluster into two halves.
245partition-halves)
246 # Get the list of ONOS nodes.
247 nodes=($(onos_nodes))
248
249 # Use a nested loop to split the network into two halves using index mod 2.
250 for i in "${!nodes[@]}"; do
251 if [[ $((i % 2)) == 0 ]]; then
252 for j in "${!nodes[@]}"; do
253 if [[ $i != $j ]] && [[ $((j % 2)) == 1 ]]; then
254 partition ${nodes[i]} ${nodes[j]}
255 partition ${nodes[j]} ${nodes[i]}
256 fi
257 done
258 fi
259 done
260 ;;
261
262# Heals all partitions on all nodes.
263heal)
264 # Get the list of ONOS nodes and heal partitions between each node in both directions.
265 nodes=$(onos_nodes)
266 for lnode in ${nodes}; do
267 for rnode in ${nodes}; do
268 if [[ ${lnode} != ${rnode} ]]; then
269 heal ${lnode} ${rnode}
270 fi
271 done
272 done
273 ;;
274
275*)
276 # If a function with the command name exists, iterate through all ONOS nodes and call the function for each node.
277 if [ -n "$(type -t $cmd)" ] && [ "$(type -t $cmd)" = function ]; then
278 nodes=$(onos_nodes)
279 for node in ${nodes}; do
280 ${cmd} ${node} "${@}"
281 done
282 exit 0
283 fi
284
285 host=$cmd
286 cmd="$1" && shift
287
288 case $cmd in
289
290 # Kills a node.
291 crash)
292 onos-kill "${host}"
293 ;;
294
Jordan Haltermanddc2b182017-07-23 01:27:47 -0700295 # Recovers a crashed node
296 recover)
297 onos-service "${host}" start
298 onos-wait-for-start "${host}"
299 ;;
300
Jordan Halterman4d32a582017-05-26 19:14:30 -0700301 # Creates a partition between the source node and a set of destination nodes.
302 partition)
303
304 # Default to ONOS nodes.
305 if [ -z "${1}" ]; then
306 nodes=$(onos_nodes)
307 else
308 nodes="${*}"
309 fi
310
311 # Iterate through all provided nodes and create partitions.
312 for node in $nodes; do
313 partition ${host} ${node}
314 done
315 ;;
316
Jordan Halterman862d4f92018-01-25 17:19:30 -0800317 # Isolates the node from all other nodes
318 isolate)
319
320 # Default to ONOS nodes.
321 if [ -z "${1}" ]; then
322 nodes=$(onos_nodes)
323 else
324 nodes="${*}"
325 fi
326
327 # Iterate through all ONOS nodes and partition the host from them.
328 for node in ${nodes}; do
329 if [[ $node != $host ]]; then
330 partition ${host} ${node}
331 partition ${node} ${host}
332 fi
333 done
334 ;;
335
Jordan Halterman4d32a582017-05-26 19:14:30 -0700336 # Splits the cluster into two halves, preserving the provided host as a bridge between the two halves.
337 partition-bridge)
338
339 # Get the list of ONOS nodes.
340 nodes=($(onos_nodes))
341
342 # Use a nested loop and split the network into two halves using index mod 2. Exclude
343 # the indicated host to ensure it can communicate with both sides of the partition.
344 for i in "${!nodes[@]}"; do
345 if [[ $((i % 2)) == 0 ]] && [[ ${nodes[i]} != ${host} ]]; then
346 for j in "${!nodes[@]}"; do
347 if [[ $i != $j ]] && [[ $((j % 2)) == 1 ]] && [[ ${nodes[j]} != ${host} ]]; then
348 partition ${nodes[i]} ${nodes[j]}
349 partition ${nodes[j]} ${nodes[i]}
350 fi
351 done
352 fi
353 done
354 ;;
355
356 # Heals a partition between the provided host and a set of ONOS nodes.
357 heal)
358
359 # Default to ONOS nodes.
360 if [ -z "${1}" ]; then
361 nodes=$(onos_nodes)
362 else
363 nodes="${*}"
364 fi
365
366 # Iterate through all provided nodes and heal partitions.
367 for node in $nodes; do
Jordan Halterman862d4f92018-01-25 17:19:30 -0800368 if [[ $node != $host ]]; then
369 heal ${host} ${node}
370 fi
Jordan Halterman4d32a582017-05-26 19:14:30 -0700371 done
372 ;;
373
374 # If we made it this far, use the command name variable to call the associated function.
375 *)
376 ${cmd} ${host} "${@}"
377 ;;
378
379 esac
380 ;;
381
382esac