Refactored startup script.
- Added onos_node.cfg as centralized config file
- To configure choice of backend (hazelcast/ramcloud)
- To configure ZooKeeper cluster member
- To create myid
- To configure default startup mode (single-node/coord-node/server-node)
- To configure RAMCloud address/port
- Added "onos.sh setup" option to create logback.xml and zoo.cfg
- A few bugfixes
Change-Id: I76fd23967087ad73ae42742288b8c548c2c2202f
diff --git a/conf/onos_node.conf b/conf/onos_node.conf
new file mode 100644
index 0000000..acc74a0
--- /dev/null
+++ b/conf/onos_node.conf
@@ -0,0 +1,38 @@
+# Name of this host (`hostname` by default)
+host.name = onosdev1
+
+# IP address of this host used for ONOS communication
+host.ip = 127.0.0.1
+
+# Role of this host
+# single-node: run ONOS with RAMCloud coordinator and server
+# coord-node : run ONOS with RAMCloud coordinator
+# server-node: run ONOS with RAMCloud server
+host.role = single-node
+
+# Back-end module to store topology/flows
+host.backend = hazelcast
+#host.backend = ramcloud
+
+# List of host name/IPs that constitute ZooKeeper cluster
+# myid will be assigned incrementally according to order of list
+zookeeper.hosts = onosdev1
+#zookeeper.hosts = onosdev1,onosdev2,onosdev3,onosdev4
+
+# Protocol used by RAMCloud coordinator (fast+udp by default)
+#ramcloud.coordinator.protocol = fast+udp
+
+# IP address of RAMCloud coordinator (host.ip by default)
+#ramcloud.coordinator.ip = 127.0.0.1
+
+# Port number of RAMCloud coordinator (12246 by default)
+#ramcloud.coordinator.port = 12246
+
+# Protocol used by RAMCloud server (fast+udp by default)
+#ramcloud.server.protocol = fast+udp
+
+# IP address of RAMCloud server (host.ip by default)
+#ramcloud.server.ip = 127.0.0.1
+
+# Port number of RAMCloud server (12242 by default)
+#ramcloud.server.port = 12242
diff --git a/conf/zoo.cfg.template b/conf/zoo.cfg.template
new file mode 100644
index 0000000..39dc329
--- /dev/null
+++ b/conf/zoo.cfg.template
@@ -0,0 +1,37 @@
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=5
+# the directory where the snapshot is stored.
+# do not use /tmp for storage, /tmp here is just
+# example sakes.
+dataDir=__DATADIR__
+# the port at which the clients will connect
+clientPort=2181
+#
+# specify all servers in the Zookeeper ensemble
+#server.1=onosgui1:2888:3888
+#server.2=onosgui2:2888:3888
+#server.3=onosgui3:2888:3888
+#server.4=onosgui4:2888:3888
+#server.5=onosgui5:2888:3888
+#server.6=onosgui6:2888:3888
+#server.7=onosgui7:2888:3888
+#server.8=onosgui8:2888:3888
+#
+#
+# Be sure to read the maintenance section of the
+# administrator guide before turning on autopurge.
+#
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
+#
+# The number of snapshots to retain in dataDir
+#autopurge.snapRetainCount=3
+# Purge task interval in hours
+# Set to "0" to disable auto purge feature
+#autopurge.purgeInterval=1
+__HOSTS__
diff --git a/onos.sh b/onos.sh
index 775ea06..17e04b1 100755
--- a/onos.sh
+++ b/onos.sh
@@ -1,37 +1,41 @@
#!/bin/bash
-# Set paths
-ONOS_HOME=`dirname $0`
+ONOS_HOME=$(cd `dirname $0`; pwd)
+ONOS_CONF_DIR=${ONOS_CONF_DIR:-${ONOS_HOME}/conf}
+ONOS_CONF_FILE=onos_node.conf
+ONOS_CONF=${ONOS_CONF:-${ONOS_CONF_DIR}/${ONOS_CONF_FILE}}
LOGDIR=${ONOS_LOGDIR:-${ONOS_HOME}/onos-logs}
ZK_DIR=${HOME}/zookeeper-3.4.5
-#ZK_CONF=${ONOS_HOME}/conf/zoo.cfg
+ZK_CONF_FILE=zoo.cfg
+ZK_CONF=${ONOS_CONF_DIR}/${ZK_CONF_FILE}
+ZK_CONF_BACKUP=${ZK_CONF}.bak
+ZK_CONF_TEMPLATE=${ONOS_CONF_DIR}/zoo.cfg.template
ZOO_LOG_DIR=${ONOS_HOME}/onos-logs
+ZK_LIB_DIR=/var/lib/zookeeper
+ZK_MY_ID=${ZK_LIB_DIR}/myid
RAMCLOUD_DIR=${HOME}/ramcloud
RAMCLOUD_HOME=${RAMCLOUD_HOME:-~/ramcloud}
RAMCLOUD_COORD_LOG=${LOGDIR}/ramcloud.coordinator.`hostname`.log
RAMCLOUD_SERVER_LOG=${LOGDIR}/ramcloud.server.`hostname`.log
-coordinatorip=`grep coordinatorIp ${ONOS_HOME}/conf/ramcloud.conf | cut -d "=" -f 2,3`
-coordinatorport=`grep coordinatorPort ${ONOS_HOME}/conf/ramcloud.conf | cut -d "=" -f 2,3`
-serverip=`grep serverIp ${ONOS_HOME}/conf/ramcloud.conf | cut -d "=" -f 2,3`
-serverport=`grep serverPort ${ONOS_HOME}/conf/ramcloud.conf | cut -d "=" -f 2,3`
-RAMCLOUD_COORDINATOR=`echo $coordinatorip","$coordinatorport`
-RAMCLOUD_SERVER=`echo $serverip","$serverport`
RAMCLOUD_BRANCH=${RAMCLOUD_BRANCH:-master}
+RAMCLOUD_COORD_PORT=12246
+RAMCLOUD_SERVER_PORT=12242
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ONOS_HOME}/lib:${RAMCLOUD_HOME}/obj.${RAMCLOUD_BRANCH}
## Because the script change dir to $ONOS_HOME, we can set ONOS_LOGBACK and LOGDIR relative to $ONOS_HOME
-ONOS_LOGBACK=${ONOS_LOGBACK:-${ONOS_HOME}/conf/logback.`hostname`.xml}
-ONOS_LOGBACK_TEMPLATE=${ONOS_HOME}/conf/logback.xml.template
+ONOS_LOGBACK=${ONOS_LOGBACK:-${ONOS_CONF_DIR}/logback.`hostname`.xml}
+ONOS_LOGBACK_BACKUP=${ONOS_LOGBACK}.bak
+ONOS_LOGBACK_TEMPLATE=${ONOS_CONF_DIR}/logback.xml.template
LOGDIR=${ONOS_LOGDIR:-${ONOS_HOME}/onos-logs}
LOGBASE=${ONOS_LOGBASE:-onos.`hostname`}
ONOS_LOG="${LOGDIR}/${LOGBASE}.log"
PCAP_LOG="${LOGDIR}/${LOGBASE}.pcap"
LOGS="$ONOS_LOG $PCAP_LOG"
-ONOS_PROPS=${ONOS_PROPS:-${ONOS_HOME}/conf/onos.properties}
+ONOS_PROPS=${ONOS_PROPS:-${ONOS_CONF_DIR}/onos.properties}
JMX_PORT=${JMX_PORT:-7189}
# Set JVM options
@@ -66,16 +70,50 @@
MVN=${MVN:-mvn -o}
+
+### Variables read from ONOS config
+if [ ! -f ${ONOS_CONF} ]; then
+ echo "${ONOS_CONF} not found."
+ exit 1
+fi
+ONOS_HOST_NAME=`grep ^host.name ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+if [ -z "${ONOS_HOST_NAME}" ]; then
+ ONOS_HOST_NAME='hostname'
+fi
+ONOS_HOST_IP=`grep ^host.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+ONOS_HOST_ROLE=`grep ^host.role ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+ONOS_HOST_BACKEND=`grep ^host.backend ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+ZK_HOSTS=`grep ^zookeeper.hosts ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_COORD_PROTOCOL=`grep ^ramcloud.coordinator.protocol ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_COORD_IP=`grep ^ramcloud.coordinator.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_COORD_PORT=`grep ^ramcloud.coordinator.port ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_SERVER_PROTOCOL=`grep ^ramcloud.server.protocol ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_SERVER_IP=`grep ^ramcloud.server.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_SERVER_PORT=`grep ^ramcloud.server.port ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+
+
+### Common functions
function usage {
- echo "Usage"
- echo " $0 start {single-node|coord-node|server-node}"
- echo " single-node: start ONOS with RAMCloud coordinator/server"
- echo " coord-node : start ONOS with RAMCloud coordinator"
- echo " server-node: start ONOS with RAMCloud server"
+ echo "Usage: setup/start/stop ONOS on this server."
+ echo " $0 setup"
+ echo " Set up ONOS node using ${ONOS_CONF_FILE}"
+ echo " - generate and replace config file of ZooKeeper."
+ echo " - create myid in ZooKeeper datadir."
+ echo " - generate and replace logback.${ONOS_HOST_NAME}.xml"
+ echo " $0 start [single-node|coord-node|server-node]"
+ echo " Start ONOS node with specific RAMCloud entities"
+ echo " - single-node: start ONOS with RAMCloud coordinator/server"
+ echo " - coord-node : start ONOS with RAMCloud coordinator"
+ echo " - server-node: start ONOS with RAMCloud server"
+ echo " * Default behavior can be defined by ${ONOS_CONF_FILE}"
echo " $0 stop"
+ echo " Stop all ONOS-related processes"
echo " $0 restart"
+ echo " Stop and start currently running ONOS-related processes"
echo " $0 status"
+ echo " Show status of ONOS-related processes"
echo " $0 {zk|rc-coord|rc-server|core} {start|stop|restart|status}"
+ echo " Control specific ONOS-related process"
}
function rotate-log {
@@ -95,8 +133,7 @@
function kill-processes {
# Kill the existing processes
local pids=$2
- local n=${#pids[*]}
- if [ $n != 0 ]; then
+ if [ ! -z "$pids" ]; then
echo -n "Stopping $1 ... "
fi
for p in ${pids}; do
@@ -107,6 +144,111 @@
done
}
+function revert-conf {
+ set -e
+
+ echo -n "ERROR occurred ... "
+ local temp_zk="${ZK_CONF}.tmp"
+
+ if [ -f "${temp_zk}" ]; then
+ local zk_file=`basename ${ZK_CONF}`
+ echo -n "reverting ${zk_file} ... "
+ rm ${temp_zk}
+ mv ${ZK_CONF_BACKUP} ${ZK_CONF}
+ fi
+
+ echo "EXIT"
+
+ if [ ! -z "$1" ]; then
+ echo $1
+ fi
+
+ set +e
+
+ exit 1
+}
+
+function create-conf {
+ # creation of zookeeper config
+
+ trap revert-conf ERR
+
+ echo -n "Creating ${ZK_CONF} ... "
+
+ local temp_zk="${ZK_CONF}.tmp"
+
+ touch ${temp_zk}
+
+ if [ -f ${ZK_CONF} ]; then
+ mv ${ZK_CONF} ${ZK_CONF_BACKUP}
+ local backup_file=`basename ${ZK_CONF_BACKUP}`
+ echo -n "backup old file to ${backup_file} ... "
+ fi
+
+ local hosts=${ZK_HOSTS}
+ if [ -z "${hosts}" ]; then
+ # assume single-node mode
+ hosts=${ONOS_HOST_NAME}
+ fi
+
+ hostarr=`echo ${hosts} | tr "," " "`
+
+ local i=1
+ local myid=
+ for host in ${hostarr}; do
+ if [ ${host} = ${ONOS_HOST_NAME} ]; then
+ myid=$i
+ break
+ fi
+ i=`expr $i + 1`
+ done
+
+ if [ -z "${myid}" ]; then
+ revert-conf "[ERROR in ${ONOS_CONF_FILE}] zookeeper.hosts must have hostname \"${ONOS_HOST_NAME}\""
+ fi
+
+ # TODO: Remove sudo.
+ # This is temporary code for the sake of compatibility with old code (which creates myid with root user).
+ sudo mv ${ZK_MY_ID} ${ZK_MY_ID}.old
+ echo ${myid} > ${ZK_MY_ID}
+
+ echo -n "myid is assigned to ${myid} ... "
+
+ while read line; do
+ if [[ $line =~ ^__HOSTS__$ ]]; then
+ i=1
+ for host in ${hostarr}; do
+ # TODO: ports might be configurable
+ local hostline="server.${i}=${host}:2888:3888"
+ echo $hostline >> "${temp_zk}"
+ i=`expr $i + 1`
+ done
+ elif [[ $line =~ __DATADIR__ ]]; then
+ echo $line | sed -e "s|__DATADIR__|${ZK_LIB_DIR}|" >> ${temp_zk}
+ else
+ echo $line >> ${temp_zk}
+ fi
+ done < ${ZK_CONF_TEMPLATE}
+ mv ${temp_zk} ${ZK_CONF}
+
+ echo "DONE"
+
+ echo -n "Creating ${ONOS_LOGBACK} ... "
+ # creation of logback config
+ if [ -f $ONOS_LOGBACK ]; then
+ local logback_file=`basename ${ONOS_LOGBACK}`
+ mv ${ONOS_LOGBACK} ${ONOS_LOGBACK_BACKUP}
+ local logback_back_file=`basename ${ONOS_LOGBACK_BACKUP}`
+ echo -n "backup old file to ${logback_back_file} ... "
+ fi
+ sed -e "s|__FILENAME__|${ONOS_LOG}|" ${ONOS_LOGBACK_TEMPLATE} > ${ONOS_LOGBACK}
+ echo "DONE"
+
+ trap - ERR
+
+ return
+}
+
### Functions related to ZooKeeper
function zk {
@@ -131,8 +273,14 @@
}
function start-zk {
- # Run Zookeeper with our configuration
echo -n "Starting Zookeeper ... "
+
+ if [ -f "${ZK_CONF}" ]; then
+ # Run Zookeeper with our configuration
+ export ZOOCFG=${ZK_CONF_FILE}
+ export ZOOCFGDIR=${ONOS_CONF_DIR}
+ fi
+
$ZK_DIR/bin/zkServer.sh start
}
@@ -141,16 +289,83 @@
}
function status-zk {
+ if [ -f ${ZK_CONF} ]; then
+ export ZOOCFG=${ZK_CONF_FILE}
+ export ZOOCFGDIR=${ONOS_CONF_DIR}
+ fi
+
$ZK_DIR/bin/zkServer.sh status
}
+function start-backend {
+ if [ "${ONOS_HOST_BACKEND}" = "ramcloud" ]; then
+ rc-coord startifdown
+ rc-server startifdown
+ fi
+}
+
+function stop-backend {
+ rcsn=`pgrep -f obj.${RAMCLOUD_BRANCH}/server | wc -l`
+ if [ $rcsn != 0 ]; then
+ rc-server stop
+ fi
+
+ rccn=`pgrep coordinator | wc -l`
+ if [ $rccn != 0 ]; then
+ rc-coord stop
+ fi
+}
+
### Functions related to RAMCloud coordinator
+function rc-coord-addr {
+ local coordproto=${RC_COORD_PROTOCOL}
+ local coordip=${RC_COORD_IP}
+ local coordport=${RC_COORD_PORT}
+
+ if [ -z "${coordproto}" ]; then
+ coordproto='fast+udp'
+ fi
+
+ if [ -z "${coordip}" ]; then
+ # assume single-node mode
+ coordip=${ONOS_HOST_IP}
+ fi
+
+ if [ -z "${coordport}" ]; then
+ # assume default port
+ coordport=${RAMCLOUD_COORD_PORT}
+ fi
+
+ echo "${coordproto}:host=${coordip},port=${coordport}"
+}
+
+function rc-server-addr {
+ local serverproto=${RC_SERVER_PROTOCOL}
+ local serverip=${RC_SERVER_IP}
+ local serverport=${RC_SERVER_PORT}
+
+ if [ -z "${serverproto}" ]; then
+ serverproto='fast+udp'
+ fi
+
+ # Normally this parameter should be null
+ if [ -z "${serverip}" ]; then
+ serverip=${ONOS_HOST_IP}
+ fi
+
+ if [ -z "${serverport}" ]; then
+ # assume default port
+ serverport=${RAMCLOUD_SERVER_PORT}
+ fi
+
+ echo "${serverproto}:host=${serverip},port=${serverport}"
+}
+
function rc-coord {
case "$1" in
start)
deldb
- cp $ONOS_HOME/conf/ramcloud.conf /tmp
stop-coord
start-coord
;;
@@ -182,16 +397,18 @@
if [ -f $RAMCLOUD_COORD_LOG ]; then
rotate-log $RAMCLOUD_COORD_LOG
fi
+
+ local coord_addr=`rc-coord-addr`
# Run ramcloud
echo -n "Starting RAMCloud coordinator ... "
- $RAMCLOUD_DIR/obj.${RAMCLOUD_BRANCH}/coordinator -L $RAMCLOUD_COORDINATOR > $RAMCLOUD_COORD_LOG 2>&1 &
+ $RAMCLOUD_DIR/obj.${RAMCLOUD_BRANCH}/coordinator -L ${coord_addr} > $RAMCLOUD_COORD_LOG 2>&1 &
echo "STARTED"
}
function stop-coord {
- kill-processes "RAMCloud coordinator" `pgrep coordinator | awk '{print $1}'`
+ kill-processes "RAMCloud coordinator" `pgrep -f obj.${RAMCLOUD_BRANCH}/coordinator`
}
function deldb {
@@ -204,7 +421,6 @@
case "$1" in
start)
deldb
- cp $ONOS_HOME/conf/ramcloud.conf /tmp
stop-server
start-server
;;
@@ -239,15 +455,18 @@
if [ -f $RAMCLOUD_SERVER_LOG ]; then
rotate-log $RAMCLOUD_SERVER_LOG
fi
+
+ local coord_addr=`rc-coord-addr`
+ local server_addr=`rc-server-addr`
# Run ramcloud
echo -n "Starting RAMCloud server ... "
- $RAMCLOUD_DIR/obj.${RAMCLOUD_BRANCH}/server -M -L $RAMCLOUD_SERVER -C $RAMCLOUD_COORDINATOR --masterServiceThreads 1 --logCleanerThreads 1 --detectFailures 0 > $RAMCLOUD_SERVER_LOG 2>&1 &
+ ${RAMCLOUD_DIR}/obj.${RAMCLOUD_BRANCH}/server -M -L ${server_addr} -C ${coord_addr} --masterServiceThreads 1 --logCleanerThreads 1 --detectFailures 0 > $RAMCLOUD_SERVER_LOG 2>&1 &
echo "STARTED"
}
function stop-server {
- kill-processes "RAMCloud server" `pgrep -f obj.${RAMCLOUD_BRANCH}/server | awk '{print $1}'`
+ kill-processes "RAMCloud server" `pgrep -f obj.${RAMCLOUD_BRANCH}/server`
}
@@ -255,6 +474,7 @@
function onos {
CPFILE=${ONOS_HOME}/.javacp.`hostname`
if [ ! -f ${CPFILE} ]; then
+ echo "ONOS core needs to be built"
${MVN} -f ${ONOS_HOME}/pom.xml compile
fi
JAVA_CP=`cat ${CPFILE}`
@@ -299,20 +519,25 @@
rotate-log ${log}
fi
done
-
- # Run ONOS
- echo -n "Starting ONOS controller ..."
-# Create default logback config file using template
+
if [ ! -f ${ONOS_LOGBACK} ]; then
- sed -e "s|__FILENAME__|${ONOS_LOG}|" ${ONOS_LOGBACK_TEMPLATE} > ${ONOS_LOGBACK}
+ echo "[WARNING] ${ONOS_LOGBACK} not found."
+ echo " Run \"\$ $0 setup\" to create."
+ exit 1
fi
+
+ JVM_OPTS="${JVM_OPTS} -Dnet.onrc.onos.datastore.backend=${ONOS_HOST_BACKEND}"
+
+ # Run ONOS
+
+ echo -n "Starting ONOS controller ..."
java ${JVM_OPTS} -Dlogback.configurationFile=${ONOS_LOGBACK} -cp ${JAVA_CP} ${MAIN_CLASS} -cf ${ONOS_PROPS} > ${LOGDIR}/${LOGBASE}.stdout 2>${LOGDIR}/${LOGBASE}.stderr &
# We need to wait a bit to find out whether starting the ONOS process succeeded
sleep 1
n=`jps -l |grep "${MAIN_CLASS}" | wc -l`
- if [ "$n" -ge "1" ]; then
+ if [ $n -ge 1 ]; then
echo " STARTED"
else
echo " FAILED"
@@ -328,13 +553,22 @@
}
+# Main
+
case "$1" in
+ setup)
+ create-conf
+ ;;
start)
- case $2 in
+ mode_parameter=${ONOS_HOST_ROLE}
+ if [ ! -z "$2" ]; then
+ mode_parameter=$2
+ fi
+
+ case "${mode_parameter}" in
single-node)
zk start
- rc-coord startifdown
- rc-server startifdown
+ start-backend
onos startifdown
;;
coord-node)
@@ -359,15 +593,7 @@
onos stop
fi
- rcsn=`pgrep -f obj.${RAMCLOUD_BRANCH}/server | wc -l`
- if [ $rcsn != 0 ]; then
- rc-server stop
- fi
-
- rccn=`pgrep coordinator | wc -l`
- if [ $rccn != 0 ]; then
- rc-coord stop
- fi
+ stop-backend
zkn=`jps -l | grep org.apache.zookeeper.server | wc -l`
if [ $zkn != 0 ]; then