Add HA test for split brain scenario

- Add option to cleanup script to reset firewall rules
- minor fixes to drivers and other HA tests
- Add test-summary script

Change-Id: I758b60ada7a3675664456b4da056ad4a6d014b30
diff --git a/TestON/bin/cleanup.sh b/TestON/bin/cleanup.sh
index dd4840f..f931eca 100755
--- a/TestON/bin/cleanup.sh
+++ b/TestON/bin/cleanup.sh
@@ -1,5 +1,10 @@
 #!/bin/bash
+# -----------------------------------------------------------------------------
 # This script will kill any TestON, ssh, and Mininet sessions that are running.
+# -----------------------------------------------------------------------------
+
+# TODO: Add help to this file, and some more options?
+#       Maybe kill/uninstall ONOS?
 sudo kill -9 `ps -ef | grep "./cli.py" | grep -v grep | awk '{print $2}'`
 sudo kill -9 `ps -ef | grep "bin/teston" | grep -v grep | awk '{print $2}'`
 sudo kill -9 `ps -ef | grep "ssh -X" | grep -v grep | awk '{print $2}'`
@@ -9,3 +14,20 @@
 sudo pkill zebra
 sudo pkill vrrpd
 sudo kill -9 `ps -ef | grep "bird" | grep -v grep | awk '{print $2}'`
+
+
+# Restore persistent firewall rules
+if [ "$1" = "-f" ]; then
+
+    OCIS=( $(env | sed -ne 's:OC[0-9]\{1,\}=\(.*\):\1 :g p' | sort -k1) )
+    if [ -z "$OCIS" ]; then
+      printf "no ONOS nodes set in your cell, Don't know where to look" >&2 && exit 0
+    fi
+
+    # TODO: Make the file configurable
+    for i in ${OCIS[@]}; do
+        echo "Restoring iptables rules on ${i}"
+        ssh sdn@$i "sudo iptables -F"
+        ssh sdn@$i "sudo iptables-restore < /etc/iptables/rules.v4"
+    done
+fi
diff --git a/TestON/bin/test-summary b/TestON/bin/test-summary
new file mode 100755
index 0000000..7832f60
--- /dev/null
+++ b/TestON/bin/test-summary
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+if [ "$#" -lt 1 ]; then
+    ls -t ~/OnosSystemTest/TestON/logs/*/*y.txt | head -1 | xargs cat | grep -E "^|FAIL"
+    exit 1
+fi
+if [ "$1" = "-f" ]; then
+    # NOTE: This is blocking
+    ls -t ~/OnosSystemTest/TestON/logs/*/*y.txt | head -1 | xargs tail -n+1 -f
+else
+    printf "usage: $(basename $0) [options]\n\n"
+    printf "Simple command to print the latest test summary file.\n"
+    printf "\noptions:\n\t--help : Displays this message and exits\n"
+    printf "\t-f     : Tails the summary file and updates as the file is written to.\n\n"
+    exit 1
+fi