Refactored startup script.

  - Added onos_node.cfg as centralized config file
    - To configure choice of backend (hazelcast/ramcloud)
    - To configure ZooKeeper cluster member
    - To create myid
    - To configure default startup mode (single-node/coord-node/server-node)
    - To configure RAMCloud address/port
  - Added "onos.sh setup" option to create logback.xml and zoo.cfg
  - A few bugfixes

Change-Id: I76fd23967087ad73ae42742288b8c548c2c2202f
diff --git a/conf/onos_node.conf b/conf/onos_node.conf
new file mode 100644
index 0000000..acc74a0
--- /dev/null
+++ b/conf/onos_node.conf
@@ -0,0 +1,38 @@
+# Name of this host (`hostname` by default)
+host.name = onosdev1
+
+# IP address of this host used for ONOS communication
+host.ip = 127.0.0.1
+
+# Role of this host
+# single-node: run ONOS with RAMCloud coordinator and server
+# coord-node : run ONOS with RAMCloud coordinator
+# server-node: run ONOS with RAMCloud server
+host.role = single-node
+
+# Back-end module to store topology/flows
+host.backend = hazelcast
+#host.backend = ramcloud
+
+# List of host name/IPs that constitute ZooKeeper cluster
+# myid will be assigned incrementally according to order of list
+zookeeper.hosts = onosdev1
+#zookeeper.hosts = onosdev1,onosdev2,onosdev3,onosdev4
+
+# Protocol used by RAMCloud coordinator (fast+udp by default)
+#ramcloud.coordinator.protocol = fast+udp
+
+# IP address of RAMCloud coordinator (host.ip by default)
+#ramcloud.coordinator.ip = 127.0.0.1
+
+# Port number of RAMCloud coordinator (12246 by default)
+#ramcloud.coordinator.port = 12246
+
+# Protocol used by RAMCloud server (fast+udp by default)
+#ramcloud.server.protocol = fast+udp
+
+# IP address of RAMCloud server (host.ip by default)
+#ramcloud.server.ip = 127.0.0.1
+
+# Port number of RAMCloud server (12242 by default)
+#ramcloud.server.port = 12242
diff --git a/conf/zoo.cfg.template b/conf/zoo.cfg.template
new file mode 100644
index 0000000..39dc329
--- /dev/null
+++ b/conf/zoo.cfg.template
@@ -0,0 +1,37 @@
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial 
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between 
+# sending a request and getting an acknowledgement
+syncLimit=5
+# the directory where the snapshot is stored.
+# do not use /tmp for storage, /tmp here is just 
+# example sakes.
+dataDir=__DATADIR__
+# the port at which the clients will connect
+clientPort=2181
+#
+# specify all servers in the Zookeeper ensemble
+#server.1=onosgui1:2888:3888
+#server.2=onosgui2:2888:3888
+#server.3=onosgui3:2888:3888
+#server.4=onosgui4:2888:3888
+#server.5=onosgui5:2888:3888
+#server.6=onosgui6:2888:3888
+#server.7=onosgui7:2888:3888
+#server.8=onosgui8:2888:3888
+#
+#
+# Be sure to read the maintenance section of the 
+# administrator guide before turning on autopurge.
+#
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
+#
+# The number of snapshots to retain in dataDir
+#autopurge.snapRetainCount=3
+# Purge task interval in hours
+# Set to "0" to disable auto purge feature
+#autopurge.purgeInterval=1
+__HOSTS__
diff --git a/onos.sh b/onos.sh
index 775ea06..17e04b1 100755
--- a/onos.sh
+++ b/onos.sh
@@ -1,37 +1,41 @@
 #!/bin/bash
-# Set paths
 
-ONOS_HOME=`dirname $0`
+ONOS_HOME=$(cd `dirname $0`; pwd)
+ONOS_CONF_DIR=${ONOS_CONF_DIR:-${ONOS_HOME}/conf}
+ONOS_CONF_FILE=onos_node.conf
+ONOS_CONF=${ONOS_CONF:-${ONOS_CONF_DIR}/${ONOS_CONF_FILE}}
 LOGDIR=${ONOS_LOGDIR:-${ONOS_HOME}/onos-logs}
 
 ZK_DIR=${HOME}/zookeeper-3.4.5
-#ZK_CONF=${ONOS_HOME}/conf/zoo.cfg
+ZK_CONF_FILE=zoo.cfg
+ZK_CONF=${ONOS_CONF_DIR}/${ZK_CONF_FILE}
+ZK_CONF_BACKUP=${ZK_CONF}.bak
+ZK_CONF_TEMPLATE=${ONOS_CONF_DIR}/zoo.cfg.template
 ZOO_LOG_DIR=${ONOS_HOME}/onos-logs
+ZK_LIB_DIR=/var/lib/zookeeper
+ZK_MY_ID=${ZK_LIB_DIR}/myid
 
 RAMCLOUD_DIR=${HOME}/ramcloud
 RAMCLOUD_HOME=${RAMCLOUD_HOME:-~/ramcloud}
 RAMCLOUD_COORD_LOG=${LOGDIR}/ramcloud.coordinator.`hostname`.log
 RAMCLOUD_SERVER_LOG=${LOGDIR}/ramcloud.server.`hostname`.log
-coordinatorip=`grep coordinatorIp ${ONOS_HOME}/conf/ramcloud.conf | cut -d "=" -f 2,3`
-coordinatorport=`grep coordinatorPort ${ONOS_HOME}/conf/ramcloud.conf | cut -d "=" -f 2,3`
-serverip=`grep serverIp ${ONOS_HOME}/conf/ramcloud.conf | cut -d "=" -f 2,3`
-serverport=`grep serverPort ${ONOS_HOME}/conf/ramcloud.conf | cut -d "=" -f 2,3`
-RAMCLOUD_COORDINATOR=`echo $coordinatorip","$coordinatorport`
-RAMCLOUD_SERVER=`echo $serverip","$serverport`
 RAMCLOUD_BRANCH=${RAMCLOUD_BRANCH:-master}
+RAMCLOUD_COORD_PORT=12246
+RAMCLOUD_SERVER_PORT=12242
 
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ONOS_HOME}/lib:${RAMCLOUD_HOME}/obj.${RAMCLOUD_BRANCH}
 
 ## Because the script change dir to $ONOS_HOME, we can set ONOS_LOGBACK and LOGDIR relative to $ONOS_HOME
-ONOS_LOGBACK=${ONOS_LOGBACK:-${ONOS_HOME}/conf/logback.`hostname`.xml}
-ONOS_LOGBACK_TEMPLATE=${ONOS_HOME}/conf/logback.xml.template
+ONOS_LOGBACK=${ONOS_LOGBACK:-${ONOS_CONF_DIR}/logback.`hostname`.xml}
+ONOS_LOGBACK_BACKUP=${ONOS_LOGBACK}.bak
+ONOS_LOGBACK_TEMPLATE=${ONOS_CONF_DIR}/logback.xml.template
 LOGDIR=${ONOS_LOGDIR:-${ONOS_HOME}/onos-logs}
 LOGBASE=${ONOS_LOGBASE:-onos.`hostname`}
 ONOS_LOG="${LOGDIR}/${LOGBASE}.log"
 PCAP_LOG="${LOGDIR}/${LOGBASE}.pcap"
 LOGS="$ONOS_LOG $PCAP_LOG"
 
-ONOS_PROPS=${ONOS_PROPS:-${ONOS_HOME}/conf/onos.properties}
+ONOS_PROPS=${ONOS_PROPS:-${ONOS_CONF_DIR}/onos.properties}
 JMX_PORT=${JMX_PORT:-7189}
 
 # Set JVM options
@@ -66,16 +70,50 @@
 
 MVN=${MVN:-mvn -o}
 
+
+### Variables read from ONOS config
+if [ ! -f ${ONOS_CONF} ]; then
+  echo "${ONOS_CONF} not found."
+  exit 1
+fi
+ONOS_HOST_NAME=`grep ^host.name ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+if [ -z "${ONOS_HOST_NAME}" ]; then
+  ONOS_HOST_NAME='hostname'
+fi
+ONOS_HOST_IP=`grep ^host.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+ONOS_HOST_ROLE=`grep ^host.role ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+ONOS_HOST_BACKEND=`grep ^host.backend ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+ZK_HOSTS=`grep ^zookeeper.hosts ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_COORD_PROTOCOL=`grep ^ramcloud.coordinator.protocol ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_COORD_IP=`grep ^ramcloud.coordinator.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_COORD_PORT=`grep ^ramcloud.coordinator.port ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_SERVER_PROTOCOL=`grep ^ramcloud.server.protocol ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_SERVER_IP=`grep ^ramcloud.server.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+RC_SERVER_PORT=`grep ^ramcloud.server.port ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+
+
+### Common functions
 function usage {
-  echo "Usage"
-  echo "  $0 start {single-node|coord-node|server-node}"
-  echo "            single-node: start ONOS with RAMCloud coordinator/server"
-  echo "            coord-node : start ONOS with RAMCloud coordinator"
-  echo "            server-node: start ONOS with RAMCloud server"
+  echo "Usage: setup/start/stop ONOS on this server."
+  echo "  $0 setup"
+  echo "    Set up ONOS node using ${ONOS_CONF_FILE}"
+  echo "      - generate and replace config file of ZooKeeper."
+  echo "      - create myid in ZooKeeper datadir."
+  echo "      - generate and replace logback.${ONOS_HOST_NAME}.xml"
+  echo "  $0 start [single-node|coord-node|server-node]"
+  echo "    Start ONOS node with specific RAMCloud entities"
+  echo "      - single-node: start ONOS with RAMCloud coordinator/server"
+  echo "      - coord-node : start ONOS with RAMCloud coordinator"
+  echo "      - server-node: start ONOS with RAMCloud server"
+  echo "      * Default behavior can be defined by ${ONOS_CONF_FILE}"
   echo "  $0 stop"
+  echo "    Stop all ONOS-related processes"
   echo "  $0 restart"
+  echo "    Stop and start currently running ONOS-related processes"
   echo "  $0 status"
+  echo "    Show status of ONOS-related processes"
   echo "  $0 {zk|rc-coord|rc-server|core} {start|stop|restart|status}"
+  echo "    Control specific ONOS-related process"
 }
 
 function rotate-log {
@@ -95,8 +133,7 @@
 function kill-processes {
   # Kill the existing processes
   local pids=$2
-  local n=${#pids[*]}
-  if [ $n != 0 ]; then
+  if [ ! -z "$pids" ]; then
     echo -n "Stopping $1 ... "
   fi
   for p in ${pids}; do
@@ -107,6 +144,111 @@
   done
 }
 
+function revert-conf {
+  set -e
+  
+  echo -n "ERROR occurred ... "
+  local temp_zk="${ZK_CONF}.tmp"
+  
+  if [ -f "${temp_zk}" ]; then
+    local zk_file=`basename ${ZK_CONF}`
+    echo -n "reverting ${zk_file} ... "
+    rm ${temp_zk}
+    mv ${ZK_CONF_BACKUP} ${ZK_CONF}
+  fi
+
+  echo "EXIT"
+  
+  if [ ! -z "$1" ]; then
+    echo $1
+  fi
+  
+  set +e
+  
+  exit 1
+}
+
+function create-conf {
+  # creation of zookeeper config
+  
+  trap revert-conf ERR
+  
+  echo -n "Creating ${ZK_CONF} ... "
+  
+  local temp_zk="${ZK_CONF}.tmp"
+  
+  touch ${temp_zk}
+  
+  if [ -f ${ZK_CONF} ]; then
+    mv ${ZK_CONF} ${ZK_CONF_BACKUP}
+    local backup_file=`basename ${ZK_CONF_BACKUP}`
+    echo -n "backup old file to ${backup_file} ... "
+  fi
+  
+  local hosts=${ZK_HOSTS}
+  if [ -z "${hosts}" ]; then
+    # assume single-node mode
+    hosts=${ONOS_HOST_NAME}
+  fi
+  
+  hostarr=`echo ${hosts} | tr "," " "`
+  
+  local i=1
+  local myid=
+  for host in ${hostarr}; do
+    if [ ${host} = ${ONOS_HOST_NAME} ]; then
+      myid=$i
+      break
+    fi
+    i=`expr $i + 1`
+  done
+  
+  if [ -z "${myid}" ]; then
+    revert-conf "[ERROR in ${ONOS_CONF_FILE}] zookeeper.hosts must have hostname \"${ONOS_HOST_NAME}\""
+  fi
+  
+  # TODO: Remove sudo.
+  # This is temporary code for the sake of compatibility with old code (which creates myid with root user).
+  sudo mv ${ZK_MY_ID} ${ZK_MY_ID}.old
+  echo ${myid} > ${ZK_MY_ID}
+  
+  echo -n "myid is assigned to ${myid} ... "
+  
+  while read line; do
+    if [[ $line =~ ^__HOSTS__$ ]]; then
+      i=1
+      for host in ${hostarr}; do
+        # TODO: ports might be configurable
+        local hostline="server.${i}=${host}:2888:3888"
+        echo $hostline >> "${temp_zk}"
+        i=`expr $i + 1`
+      done
+    elif [[ $line =~ __DATADIR__ ]]; then
+      echo $line | sed -e "s|__DATADIR__|${ZK_LIB_DIR}|" >> ${temp_zk}
+    else
+      echo $line >> ${temp_zk}
+    fi
+  done < ${ZK_CONF_TEMPLATE}
+  mv ${temp_zk} ${ZK_CONF}
+  
+  echo "DONE"
+
+  echo -n "Creating ${ONOS_LOGBACK} ... "
+  # creation of logback config
+  if [ -f $ONOS_LOGBACK ]; then
+    local logback_file=`basename ${ONOS_LOGBACK}`
+    mv ${ONOS_LOGBACK} ${ONOS_LOGBACK_BACKUP}
+    local logback_back_file=`basename ${ONOS_LOGBACK_BACKUP}`
+    echo -n "backup old file to ${logback_back_file} ... "
+  fi
+  sed -e "s|__FILENAME__|${ONOS_LOG}|" ${ONOS_LOGBACK_TEMPLATE} > ${ONOS_LOGBACK}
+  echo "DONE"
+  
+  trap - ERR
+  
+  return
+}
+
 
 ### Functions related to ZooKeeper
 function zk {
@@ -131,8 +273,14 @@
 }
 
 function start-zk {
-  # Run Zookeeper with our configuration
   echo -n "Starting Zookeeper ... "
+  
+  if [ -f "${ZK_CONF}" ]; then
+    # Run Zookeeper with our configuration
+    export ZOOCFG=${ZK_CONF_FILE}
+    export ZOOCFGDIR=${ONOS_CONF_DIR}
+  fi
+  
   $ZK_DIR/bin/zkServer.sh start
 }
 
@@ -141,16 +289,83 @@
 }
 
 function status-zk {
+  if [ -f ${ZK_CONF} ]; then
+    export ZOOCFG=${ZK_CONF_FILE}
+    export ZOOCFGDIR=${ONOS_CONF_DIR}
+  fi
+  
   $ZK_DIR/bin/zkServer.sh status
 }
 
 
+function start-backend {
+  if [ "${ONOS_HOST_BACKEND}" = "ramcloud" ]; then
+    rc-coord startifdown
+    rc-server startifdown
+  fi
+}
+
+function stop-backend {
+  rcsn=`pgrep -f obj.${RAMCLOUD_BRANCH}/server | wc -l`
+  if [ $rcsn != 0 ]; then
+    rc-server stop
+  fi
+  
+  rccn=`pgrep coordinator | wc -l`
+  if [ $rccn != 0 ]; then
+    rc-coord stop
+  fi
+}
+
 ### Functions related to RAMCloud coordinator
+function rc-coord-addr {
+  local coordproto=${RC_COORD_PROTOCOL}
+  local coordip=${RC_COORD_IP}
+  local coordport=${RC_COORD_PORT}
+
+  if [ -z "${coordproto}" ]; then
+    coordproto='fast+udp'
+  fi
+
+  if [ -z "${coordip}" ]; then
+    # assume single-node mode
+    coordip=${ONOS_HOST_IP}
+  fi
+  
+  if [ -z "${coordport}" ]; then
+    # assume default port
+    coordport=${RAMCLOUD_COORD_PORT}
+  fi
+  
+  echo "${coordproto}:host=${coordip},port=${coordport}"
+}
+
+function rc-server-addr {
+  local serverproto=${RC_SERVER_PROTOCOL}
+  local serverip=${RC_SERVER_IP}
+  local serverport=${RC_SERVER_PORT}
+
+  if [ -z "${serverproto}" ]; then
+    serverproto='fast+udp'
+  fi
+
+  # Normally this parameter should be null
+  if [ -z "${serverip}" ]; then
+    serverip=${ONOS_HOST_IP}
+  fi
+  
+  if [ -z "${serverport}" ]; then
+    # assume default port
+    serverport=${RAMCLOUD_SERVER_PORT}
+  fi
+  
+  echo "${serverproto}:host=${serverip},port=${serverport}"
+}
+
 function rc-coord {
   case "$1" in
     start)
       deldb
-      cp $ONOS_HOME/conf/ramcloud.conf /tmp
       stop-coord
       start-coord
       ;;
@@ -182,16 +397,18 @@
   if [ -f $RAMCLOUD_COORD_LOG ]; then
     rotate-log $RAMCLOUD_COORD_LOG
   fi
+  
+  local coord_addr=`rc-coord-addr`
 
   # Run ramcloud 
   echo -n "Starting RAMCloud coordinator ... "
-  $RAMCLOUD_DIR/obj.${RAMCLOUD_BRANCH}/coordinator -L $RAMCLOUD_COORDINATOR > $RAMCLOUD_COORD_LOG 2>&1 &
+  $RAMCLOUD_DIR/obj.${RAMCLOUD_BRANCH}/coordinator -L ${coord_addr} > $RAMCLOUD_COORD_LOG 2>&1 &
   echo "STARTED"
 }
 
 
 function stop-coord {
-  kill-processes "RAMCloud coordinator" `pgrep coordinator | awk '{print $1}'`
+  kill-processes "RAMCloud coordinator" `pgrep -f obj.${RAMCLOUD_BRANCH}/coordinator`
 }
 
 function deldb {
@@ -204,7 +421,6 @@
   case "$1" in
     start)
       deldb
-      cp $ONOS_HOME/conf/ramcloud.conf /tmp
       stop-server
       start-server
       ;;
@@ -239,15 +455,18 @@
   if [ -f $RAMCLOUD_SERVER_LOG ]; then
     rotate-log $RAMCLOUD_SERVER_LOG
   fi
+  
+  local coord_addr=`rc-coord-addr`
+  local server_addr=`rc-server-addr`
 
   # Run ramcloud
   echo -n "Starting RAMCloud server ... "
-  $RAMCLOUD_DIR/obj.${RAMCLOUD_BRANCH}/server -M -L $RAMCLOUD_SERVER -C $RAMCLOUD_COORDINATOR --masterServiceThreads 1 --logCleanerThreads 1 --detectFailures 0 > $RAMCLOUD_SERVER_LOG 2>&1 &
+  ${RAMCLOUD_DIR}/obj.${RAMCLOUD_BRANCH}/server -M -L ${server_addr} -C ${coord_addr} --masterServiceThreads 1 --logCleanerThreads 1 --detectFailures 0 > $RAMCLOUD_SERVER_LOG 2>&1 &
   echo "STARTED"
 }
 
 function stop-server {
-  kill-processes "RAMCloud server" `pgrep -f obj.${RAMCLOUD_BRANCH}/server | awk '{print $1}'`
+  kill-processes "RAMCloud server" `pgrep -f obj.${RAMCLOUD_BRANCH}/server`
 }
 
 
@@ -255,6 +474,7 @@
 function onos {
   CPFILE=${ONOS_HOME}/.javacp.`hostname`
   if [ ! -f ${CPFILE} ]; then
+    echo "ONOS core needs to be built"
     ${MVN} -f ${ONOS_HOME}/pom.xml compile
   fi
   JAVA_CP=`cat ${CPFILE}`
@@ -299,20 +519,25 @@
       rotate-log ${log}
     fi
   done
-
-  # Run ONOS
-  echo -n "Starting ONOS controller ..."
-# Create default logback config file using template
+  
   if [ ! -f ${ONOS_LOGBACK} ]; then
-    sed -e "s|__FILENAME__|${ONOS_LOG}|" ${ONOS_LOGBACK_TEMPLATE} > ${ONOS_LOGBACK}
+    echo "[WARNING] ${ONOS_LOGBACK} not found."
+    echo "          Run \"\$ $0 setup\" to create."
+    exit 1
   fi
+
+  JVM_OPTS="${JVM_OPTS} -Dnet.onrc.onos.datastore.backend=${ONOS_HOST_BACKEND}"
+  
+  # Run ONOS
+  
+  echo -n "Starting ONOS controller ..."
   java ${JVM_OPTS} -Dlogback.configurationFile=${ONOS_LOGBACK} -cp ${JAVA_CP} ${MAIN_CLASS} -cf ${ONOS_PROPS} > ${LOGDIR}/${LOGBASE}.stdout 2>${LOGDIR}/${LOGBASE}.stderr &
   
   # We need to wait a bit to find out whether starting the ONOS process succeeded
   sleep 1
   
   n=`jps -l |grep "${MAIN_CLASS}" | wc -l`
-  if [ "$n" -ge "1" ]; then
+  if [ $n -ge 1 ]; then
     echo " STARTED"
   else
     echo " FAILED"
@@ -328,13 +553,22 @@
 }
 
 
+# Main
+
 case "$1" in
+  setup)
+    create-conf
+    ;;
   start)
-    case $2 in
+    mode_parameter=${ONOS_HOST_ROLE}
+    if [ ! -z "$2" ]; then
+      mode_parameter=$2
+    fi
+    
+    case "${mode_parameter}" in
       single-node)
         zk start
-        rc-coord startifdown
-        rc-server startifdown
+        start-backend
         onos startifdown
         ;;
       coord-node)
@@ -359,15 +593,7 @@
       onos stop
     fi
     
-    rcsn=`pgrep -f obj.${RAMCLOUD_BRANCH}/server | wc -l`
-    if [ $rcsn != 0 ]; then
-      rc-server stop
-    fi
-    
-    rccn=`pgrep coordinator | wc -l`
-    if [ $rccn != 0 ]; then
-      rc-coord stop
-    fi
+    stop-backend
     
     zkn=`jps -l | grep org.apache.zookeeper.server | wc -l`
     if [ $zkn != 0 ]; then