[ONOS-5999] Automate DELTA security tests
Change-Id: I91e944a4b3859e792d986b028eeb8998b41dbb94
diff --git a/DELTA/README.md b/DELTA/README.md
new file mode 100644
index 0000000..8a7cc5d
--- /dev/null
+++ b/DELTA/README.md
@@ -0,0 +1,15 @@
+The scripts in this directory automates DELTA security tests against ONOS.
+
+run-DELTA.sh automates DELTA tests (All-In-One Single Machine mode). It installs DELTA and all dependencies; brings up VMs and configures the network; triggers tests using a python script (run-DELTA.py); cleans up the environment after tests are done.
+Usage of run-DELTA.sh:
+-h show help message
+-d initialize DELTA repo, build and configure DELTA
+-v destroy and reinstall vagrant VMs
+-o <name> specify name of ONOS nightly build file
+-p <path> specify path of DELTA
+
+run-DELTA.py uses pexpect to talk to DELTA manager and triggers all CONTROL_PLANE_OF and ADVANCED test cases. It also reads the DELTA log and prints the results for each test case. run-DELTA.py can take one argument for specifying DELTA directory.
+
+Note: run-DELTA.sh and run-DELTA.py should be put into the same folder.
+
+For more information of DELTA, please go to https://github.com/OpenNetworkingFoundation/delta
diff --git a/DELTA/run-DELTA.py b/DELTA/run-DELTA.py
new file mode 100755
index 0000000..1bfd7a3
--- /dev/null
+++ b/DELTA/run-DELTA.py
@@ -0,0 +1,116 @@
+#!/usr/bin/python
+'''
+This script uses pexpect to talk to DELTA manager and triggers all CONTROL_PLANE_OF and
+ADVANCED test cases. It also reads the DELTA log and prints the results for each case
+'''
+
+import sys
+import pexpect
+import time
+import datetime
+
+DELTA_DIR = '/home/sdn/DELTA'
+DELTA_LOG = 'delta.log'
+RESULT_FILE = 'summary.txt'
+LOG_CHECK_INTERVAL = 10
+# TODO: get attack codes from DELTA CLI
+CODES = ['2.1.010','2.1.020','2.1.030','2.1.040','2.1.050','2.1.060','2.1.070','2.1.071','2.1.072','2.1.073','2.1.080','3.1.010','3.1.020','3.1.030','3.1.040','3.1.050','3.1.060','3.1.070','3.1.080','3.1.090','3.1.100','3.1.110','3.1.120','3.1.130','3.1.140','3.1.150','3.1.160','3.1.170','3.1.180','3.1.190','3.1.200']
+CODE_TO_SKIP = ['3.1.090','3.1.160']
+# Timeout for each test case
+TIMEOUT = 1800
+
+def triggerTest( handle, code ):
+ testAvailable = True
+ print datetime.datetime.now(), "Starting test", code
+ # TODO: expect Exceptions thrown by DELTA
+ i = handle.expect( ['Command>', pexpect.EOF, pexpect.TIMEOUT], 60 )
+ if i != 0:
+ print "pexpect EOF or TIMEOUT, exiting..."
+ return -1
+ time.sleep(0.5)
+
+ handle.sendline( 'k' )
+ i = handle.expect( ['Select the attack code>', pexpect.EOF, pexpect.TIMEOUT], 60 )
+ if i != 0:
+ print "pexpect EOF or TIMEOUT, exiting..."
+ return -1
+ time.sleep(0.5)
+
+ handle.sendline( code )
+ i = handle.expect( ['not available', 'Press ENTER key to continue..', pexpect.EOF, pexpect.TIMEOUT], 60 )
+ if i == 0:
+ testAvailable = False
+ elif i == 1:
+ testAvailable = True
+ else:
+ print "pexpect EOF or TIMEOUT, exiting..."
+ return -1
+ time.sleep(0.5)
+
+ handle.sendline( '' )
+ if not testAvailable:
+ print "Test", code, "is not available"
+ return 0
+
+ return 1
+
+def waitForTest( code ):
+ startTime = time.time()
+ while True:
+ if time.time() - startTime > TIMEOUT:
+ print "Test timeout, exiting..."
+ return -1
+ time.sleep( LOG_CHECK_INTERVAL )
+ log = open( DELTA_LOG ).read()
+ log = log.split( code )
+ if len( log ) == 1:
+ pass
+ elif "done" in log[-1]:
+ try:
+ testName = log[1].split( ' - ' )[1]
+ except IndexError:
+ print "Error getting test name"
+ testName = "Unknown Test Name"
+ result = "UNKNOWN"
+ if "FAIL" in log[-1]:
+ result = "FAIL"
+ elif "PASS" in log[-1]:
+ result = "PASS"
+ print datetime.datetime.now(), "Test result:", result, "Time taken:", time.time() - startTime, "seconds"
+ resultFile = open( RESULT_FILE, 'a' )
+ resultFile.write( code + " " + testName + ": " + result + "\n" )
+ resultFile.close()
+ return 1
+ else:
+ pass
+
+def runTests():
+ resultFile = open( RESULT_FILE, 'w' )
+ resultFile.write( "Test started on " + str(datetime.datetime.now())+"\n" )
+ resultFile.close()
+ handle=pexpect.spawn( 'java -jar ' + DELTA_DIR + '/manager/target/delta-manager-1.0-SNAPSHOT-jar-with-dependencies.jar ' + DELTA_DIR + '/tools/config/manager.cfg' )
+ for code in CODES:
+ # Skip some broken cases
+ if code in CODE_TO_SKIP:
+ continue
+ triggerResult = triggerTest( handle, code )
+ # pexpect failures
+ if triggerResult == -1:
+ return
+ # Test not available
+ elif triggerResult == 0:
+ continue
+ testResult = waitForTest( code )
+ # Test timed out
+ if testResult == -1:
+ break
+ # Exit DELTA
+ print "All tests done, exiting DELTA"
+ i = handle.expect( ['Command>', pexpect.EOF, pexpect.TIMEOUT], 60 )
+ handle.sendline( 'q' )
+
+if __name__ == '__main__':
+ if len( sys.argv ) >= 2:
+ DELTA_DIR = sys.argv[1]
+ print 'DELTA directory is', DELTA_DIR
+ runTests()
diff --git a/DELTA/run-DELTA.sh b/DELTA/run-DELTA.sh
new file mode 100755
index 0000000..c1f3aab
--- /dev/null
+++ b/DELTA/run-DELTA.sh
@@ -0,0 +1,157 @@
+#!/bin/bash
+# This script automates DELTA tests (All-In-One Single Machine mode). It
+# installs DELTA and all dependencies; brings up VMs and configures the
+# network; triggers tests using a python script (run-DELTA.py); cleans
+# up the environment after tests are done.
+# Note: run-DELTA.py and this script should be put in the same folder
+
+set -e
+
+DELTA_PATH=/home/sdn
+
+# Install DELTA and dependencies
+function init_delta() {
+ echo "*** Initialize DELTA ***"
+ if [ ! -d "$DELTA_DIR" ]
+ then
+ echo "Downloading DELTA..."
+ (cd $DELTA_PATH && git clone https://github.com/OpenNetworkingFoundation/DELTA.git)
+ cd $DELTA_DIR
+ echo "Installing DELTA dependencies..."
+ ./tools/dev/delta-setup/delta-setup-devenv-ubuntu
+ echo "Building DELTA..."
+ source ./tools/dev/delta-setup/bash_profile
+ mvn clean install
+ echo "Installing virtualbox and vagrant..."
+ ./tools/dev/delta-setup/delta-setup-vms-ubuntu
+ # TODO: change ./tools/config/manager.cfg
+ cd -
+ fi
+}
+
+# Bring up VMs and config network
+function setup_vm() {
+ echo "*** Setting up VMs ***"
+ echo "Bringing up VMs..."
+ cd $DELTA_DIR"/tools/dev/vagrant/"
+ vagrant up >/dev/null 2>&1
+ echo "Checking if all VMs are up..."
+ vagrant status | grep controller | grep running
+ vagrant status | grep channel | grep running
+ vagrant status | grep mininet | grep running
+ echo "Setting up NAT network..."
+ VBoxManage natnetwork add --netname NatNetwork --network 10.0.2.0/24 --enable --dhcp on
+ VBoxManage controlvm $(VBoxManage list vms | grep mininet | awk '{print $1}' | sed 's/"//g') nic1 natnetwork NatNetwork
+ VBoxManage controlvm $(VBoxManage list vms | grep mininet | awk '{print $1}' | sed 's/"//g') nicpromisc1 allow-all
+ source ../delta-setup/bash_profile
+ if [[ $INIT_VM -eq 1 ]]
+ then
+ INIT_VM=0
+ echo "Setting up passwordless login..."
+ for vm in $DELTA_APP $DELTA_CHANNEL $DELTA_HOST
+ do
+ IFS=@ read name ip <<< $vm
+ ssh-keygen -f "$HOME/.ssh/known_hosts" -R $ip
+ cat ~/.ssh/id_rsa.pub | sshpass -p "vagrant" ssh -o StrictHostKeyChecking=no $vm 'cat >> .ssh/authorized_keys'
+ done
+ echo "Setting up DELTA_APP VM..."
+ ssh $DELTA_APP "sudo echo 'export JAVA_HOME=/usr/lib/jvm/java-8-oracle' | sudo tee --append /etc/environment;\
+ sudo echo 'export ONOS_APPS=drivers,openflow,proxyarp,mobility,fwd' | sudo tee --append /etc/environment"
+ echo "Copying files to VMs..."
+ ../onos-setup/onos-1.6.0-scp
+ ../delta-setup/delta-agents-scp
+ fi
+ echo "Setting up ONOS..."
+ ssh $DELTA_APP "echo 'Downloading ONOS nightly build...';\
+ set -e;\
+ wget -c http://downloads.onosproject.org/nightly/$NIGHTLY_FILE_NAME.tar.gz >/dev/null 2>&1;\
+ tar xzf $NIGHTLY_FILE_NAME.tar.gz;\
+ rm $NIGHTLY_FILE_NAME.tar.gz;\
+ if [ -d onos ]; then rm -r onos; fi;\
+ mv onos-*-SNAPSHOT onos;\
+ cp delta-agent-app-onos-1.6-1.0-SNAPSHOT.jar onos/apache-karaf-*/deploy/"
+ cd -
+}
+
+# Run DELTA tests
+function run_test() {
+ echo "*** Run tests ***"
+ cd $DELTA_DIR
+ source ./tools/dev/delta-setup/bash_profile
+ cd -
+ ./run-DELTA.py $DELTA_DIR
+}
+
+# Clean up
+function teardown_vm() {
+ echo "*** Tearing down VMs ***"
+ echo "Killing DELTA..."
+ sudo kill -9 $(ps -ef | grep delta-manager | grep -v grep | awk '{print $2}')
+ echo "Deleting NAT network..."
+ VBoxManage controlvm $(VBoxManage list vms | grep mininet | awk '{print $1}' | sed 's/"//g') nic1 nat || echo "nic1 of mininet VM not reset"
+ VBoxManage natnetwork remove --netname NatNetwork || echo "NAT network not removed"
+ echo "Bringing down VMs..."
+ cd $DELTA_DIR"/tools/dev/vagrant/"
+ if [[ $INIT_VM -eq 1 ]]
+ then
+ vagrant destroy -f
+ echo "Checking if all VMs are gone..."
+ vagrant status | grep controller | grep "not created"
+ vagrant status | grep channel | grep "not created"
+ vagrant status | grep mininet | grep "not created"
+ else
+ vagrant halt
+ echo "Checking if all VMs are down..."
+ vagrant status | grep controller | grep poweroff
+ vagrant status | grep channel | grep poweroff
+ vagrant status | grep mininet | grep poweroff
+ fi
+ cd -
+}
+
+INIT_DELTA=0
+INIT_VM=0
+NIGHTLY_FILE_NAME="onos-1.10.0.20170223-NIGHTLY"
+
+while getopts "hdvo:p:" opt; do
+ case ${opt} in
+ h )
+ echo "Usage:"
+ echo "-h show this help message"
+ echo "-d initialize DELTA repo, build and configure DELTA"
+ echo "-v destroy and reinstall vagrant VMs"
+ echo "-o <name> specify name of ONOS nightly build file"
+ echo "-p <path> specify path of DELTA"
+ exit 0
+ ;;
+ d ) INIT_DELTA=1
+ ;;
+ v ) INIT_VM=1
+ ;;
+ o ) NIGHTLY_FILE_NAME=$OPTARG
+ ;;
+ p ) DELTA_PATH=$OPTARG
+ ;;
+ \? ) echo "Invalid option: -$OPTARG"
+ exit 1
+ ;;
+ esac
+done
+
+DELTA_DIR=$DELTA_PATH"/DELTA"
+
+teardown_vm
+
+if [[ $INIT_DELTA -eq 1 ]]
+then
+ init_delta
+fi
+
+setup_vm
+
+run_test
+
+teardown_vm
+
+echo "Done"
+exit 0