Moved template files to separate directory.

Modified onos.sh and onos_node.conf to be able to config hazelcast.xml.

Modified some of onos.sh parameters to use default value.

Modified ZooKeeper's log dir to be configurable.

Refactored onos.sh.
 - Aggregated parameters' default value to declaration point.
 - Segregate common routines to functions.

Change-Id: Ia1e83fc9666c58b8897d4e2daef0326c400f48bd
diff --git a/conf/onos_node.conf b/conf/onos_node.conf
index acc74a0..499283b 100644
--- a/conf/onos_node.conf
+++ b/conf/onos_node.conf
@@ -1,5 +1,5 @@
 # Name of this host (`hostname` by default)
-host.name = onosdev1
+#host.name = onosdev1
 
 # IP address of this host used for ONOS communication
 host.ip = 127.0.0.1
@@ -14,9 +14,8 @@
 host.backend = hazelcast
 #host.backend = ramcloud
 
-# List of host name/IPs that constitute ZooKeeper cluster
+# List of host name/IPs that constitute ZooKeeper cluster (only current host by default)
 # myid will be assigned incrementally according to order of list
-zookeeper.hosts = onosdev1
 #zookeeper.hosts = onosdev1,onosdev2,onosdev3,onosdev4
 
 # Protocol used by RAMCloud coordinator (fast+udp by default)
@@ -36,3 +35,15 @@
 
 # Port number of RAMCloud server (12242 by default)
 #ramcloud.server.port = 12242
+
+# List of hostname/ip[:port] which forms Hazelcast grid
+# If this value is empty, Hazelcast will be set to muticast mode.
+# Inversely, if this value is set, multicast settings will be ignored.
+#hazelcast.tcp-ip.members = 192.168.56.11, 192.168.56.12, 192.168.56.13, 192.168.56.14
+
+# Multicast address used by Hazelcast. (224.2.2.3 by default)
+#hazelcast.multicast.group = 224.2.2.3
+
+# Multicast port used by Hazelcast. (54327 by default)
+#hazelcast.multicast.port = 54327
+
diff --git a/conf/template/hazelcast.xml.template b/conf/template/hazelcast.xml.template
new file mode 100644
index 0000000..77db87c
--- /dev/null
+++ b/conf/template/hazelcast.xml.template
@@ -0,0 +1,104 @@
+<hazelcast xsi:schemaLocation="http://www.hazelcast.com/schema/config
+	http://www.hazelcast.com/schema/config/hazelcast-config-3.0.xsd"
+	xmlns="http://www.hazelcast.com/schema/config"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+  <!-- Configuration of maps used as a data store. -->
+  <map name="datastore://*">
+    <!-- must use 'sync' backup to imitate other data store -->
+    <backup-count>3</backup-count>
+    <async-backup-count>0</async-backup-count>
+    <!-- must be false for strong consistency -->
+    <read-backup-data>false</read-backup-data>
+    <!-- near cache must not be used -->
+  </map>
+
+  <map name="*">
+
+    <!--
+    Number of sync-backups. If 1 is set as the backup-count for example,
+    then all entries of the map will be copied to another JVM for
+    ail-safety. Valid numbers are 0 (no backup), 1, 2, 3.
+    -->
+    <backup-count>0</backup-count>
+
+    <!--
+    Number of async-backups. If 1 is set as the backup-count for example,
+    then all entries of the map will be copied to another JVM for
+    fail-safety. Valid numbers are 0 (no backup), 1, 2, 3.
+    -->
+    <async-backup-count>3</async-backup-count>
+
+    <!--
+    Can we read the local backup entries? Default value is false for
+    strong consistency. Being able to read backup data will give you
+    greater performance.
+    -->
+    <read-backup-data>true</read-backup-data>
+
+    <near-cache>
+      <!--
+	  Maximum size of the near cache. When max size is reached,
+	  cache is evicted based on the policy defined.
+	  Any integer between 0 and Integer.MAX_VALUE. 0 means
+	  Integer.MAX_VALUE. Default is 0.
+      -->
+      <max-size>0</max-size>
+      <!--
+	  Maximum number of seconds for each entry to stay in the near cache.
+	  Entries that are older than <time-to-live-seconds> will get
+	  automatically evicted from the near cache.
+	  Any integer between 0 and Integer.MAX_VALUE. 0 means infinite.
+	  Default is 0.
+      -->
+      <time-to-live-seconds>0</time-to-live-seconds>
+
+      <!--
+	  Maximum number of seconds each entry can stay in the near cache as
+	  untouched (not-read).
+	  Entries that are not read (touched) more than <max-idle-seconds>
+	  value will get removed from the near cache.
+	  Any integer between 0 and Integer.MAX_VALUE. 0 means
+	  Integer.MAX_VALUE. Default is 0.
+      -->
+      <max-idle-seconds>0</max-idle-seconds>
+
+      <!--
+	  Valid values are:
+	  NONE (no extra eviction, <time-to-live-seconds> may still apply),
+	  LRU (Least Recently Used),
+	  LFU (Least Frequently Used).
+	  LRU is the default.
+	  Regardless of the eviction policy used, <time-to-live-seconds> will
+	  still apply.
+      -->
+      <eviction-policy>NONE</eviction-policy>
+
+      <!--
+	  Should the cached entries get evicted if the entries are changed
+	  (updated or removed).
+	  true of false. Default is true.
+      -->
+      <invalidate-on-change>true</invalidate-on-change>
+
+    </near-cache>
+  </map>
+
+  <topic name="*">
+    <global-ordering-enabled>false</global-ordering-enabled>
+  </topic>
+
+  <network>
+    <port auto-increment="true">5701</port>
+    <join>
+__HC_NETWORK__
+      <aws enabled="false">
+      </aws>
+    </join>
+  </network>
+
+  <properties>
+    <property name="hazelcast.logging.type">slf4j</property>
+    <property name="hazelcast.version.check.enabled">false</property>
+  </properties>
+</hazelcast>
diff --git a/conf/logback.xml.template b/conf/template/logback.xml.template
similarity index 100%
rename from conf/logback.xml.template
rename to conf/template/logback.xml.template
diff --git a/conf/zoo.cfg.template b/conf/template/zoo.cfg.template
similarity index 100%
rename from conf/zoo.cfg.template
rename to conf/template/zoo.cfg.template
diff --git a/onos.sh b/onos.sh
index 033c7cc..10ea8ce 100755
--- a/onos.sh
+++ b/onos.sh
@@ -14,7 +14,16 @@
 #####################################################
 
 
-### Variables read from ONOS config file ###
+# read-conf {filename} {parameter name} [default value]
+function read-conf {
+  local value=`grep ^${2} ${1} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+  if [ -z "${value}" ]; then
+    echo $3
+  else
+    echo ${value}
+  fi
+}
+
 ONOS_HOME=${ONOS_HOME:-$(cd `dirname $0`; pwd)}
 ONOS_CONF_DIR=${ONOS_CONF_DIR:-${ONOS_HOME}/conf}
 ONOS_CONF=${ONOS_CONF:-${ONOS_CONF_DIR}/onos_node.conf}
@@ -23,49 +32,54 @@
   echo "${ONOS_CONF} not found."
   exit 1
 fi
-ONOS_HOST_NAME=`grep ^host.name ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-if [ -z "${ONOS_HOST_NAME}" ]; then
-  ONOS_HOST_NAME=`hostname`
-fi
-ONOS_HOST_IP=`grep ^host.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-ONOS_HOST_ROLE=`grep ^host.role ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-ONOS_HOST_BACKEND=`grep ^host.backend ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-ZK_HOSTS=`grep ^zookeeper.hosts ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-RC_COORD_PROTOCOL=`grep ^ramcloud.coordinator.protocol ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-RC_COORD_IP=`grep ^ramcloud.coordinator.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-RC_COORD_PORT=`grep ^ramcloud.coordinator.port ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-RC_SERVER_PROTOCOL=`grep ^ramcloud.server.protocol ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-RC_SERVER_IP=`grep ^ramcloud.server.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
-RC_SERVER_PORT=`grep ^ramcloud.server.port ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'`
+
+
+### Variables read from ONOS config file ###
+ONOS_HOST_NAME=$(read-conf ${ONOS_CONF}     host.name                     `hostname`)
+ONOS_HOST_IP=$(read-conf ${ONOS_CONF}       host.ip)
+ONOS_HOST_ROLE=$(read-conf ${ONOS_CONF}     host.role)
+ONOS_HOST_BACKEND=$(read-conf ${ONOS_CONF}  host.backend)
+ZK_HOSTS=$(read-conf ${ONOS_CONF}           zookeeper.hosts               ${ONOS_HOST_NAME})
+RC_COORD_PROTOCOL=$(read-conf ${ONOS_CONF}  ramcloud.coordinator.protocol "fast+udp")
+RC_COORD_IP=$(read-conf ${ONOS_CONF}        ramcloud.coordinator.ip       ${ONOS_HOST_IP})
+RC_COORD_PORT=$(read-conf ${ONOS_CONF}      ramcloud.coordinator.port     12246)
+RC_SERVER_PROTOCOL=$(read-conf ${ONOS_CONF} ramcloud.server.protocol      "fast+udp")
+RC_SERVER_IP=$(read-conf ${ONOS_CONF}       ramcloud.server.ip            ${ONOS_HOST_IP})
+RC_SERVER_PORT=$(read-conf ${ONOS_CONF}     ramcloud.server.port          12242)
+HC_TCPIP_MEMBERS=$(read-conf ${ONOS_CONF}   hazelcast.tcp-ip.members)
+HC_MULTICAST_GROUP=$(read-conf ${ONOS_CONF} hazelcast.multicast.group     "224.2.2.3")
+HC_MULTICAST_PORT=$(read-conf ${ONOS_CONF}  hazelcast.multicast.port      54327)
 ############################################
 
 
 ############## Other variables #############
+ONOS_TEMPLATE_DIR=${ONOS_CONF_DIR}/template
+
 LOGDIR=${ONOS_LOGDIR:-${ONOS_HOME}/onos-logs}
 
 ZK_DIR=${HOME}/zookeeper-3.4.5
 ZK_CONF_FILE=zoo.cfg
 ZK_CONF=${ONOS_CONF_DIR}/${ZK_CONF_FILE}
-ZK_CONF_BACKUP=${ZK_CONF}.bak
-ZK_CONF_TEMPLATE=${ONOS_CONF_DIR}/zoo.cfg.template
-ZOO_LOG_DIR=${ONOS_HOME}/onos-logs
+ZK_CONF_TEMPLATE=${ONOS_TEMPLATE_DIR}/zoo.cfg.template
+ZK_LOG_DIR=${ONOS_HOME}/onos-logs
 ZK_LIB_DIR=/var/lib/zookeeper
 ZK_MY_ID=${ZK_LIB_DIR}/myid
 
+HC_CONF=${ONOS_CONF_DIR}/hazelcast.xml
+HC_CONF_TEMPLATE=${ONOS_TEMPLATE_DIR}/hazelcast.xml.template
+
 RAMCLOUD_DIR=${HOME}/ramcloud
 RAMCLOUD_HOME=${RAMCLOUD_HOME:-~/ramcloud}
 RAMCLOUD_COORD_LOG=${LOGDIR}/ramcloud.coordinator.${ONOS_HOST_NAME}.log
 RAMCLOUD_SERVER_LOG=${LOGDIR}/ramcloud.server.${ONOS_HOST_NAME}.log
 RAMCLOUD_BRANCH=${RAMCLOUD_BRANCH:-master}
-RAMCLOUD_COORD_PORT=12246
-RAMCLOUD_SERVER_PORT=12242
 
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ONOS_HOME}/lib:${RAMCLOUD_HOME}/obj.${RAMCLOUD_BRANCH}
 
 ## Because the script change dir to $ONOS_HOME, we can set ONOS_LOGBACK and LOGDIR relative to $ONOS_HOME
 ONOS_LOGBACK=${ONOS_LOGBACK:-${ONOS_CONF_DIR}/logback.${ONOS_HOST_NAME}.xml}
 ONOS_LOGBACK_BACKUP=${ONOS_LOGBACK}.bak
-ONOS_LOGBACK_TEMPLATE=${ONOS_CONF_DIR}/logback.xml.template
+ONOS_LOGBACK_TEMPLATE=${ONOS_TEMPLATE_DIR}/logback.xml.template
 LOGDIR=${ONOS_LOGDIR:-${ONOS_HOME}/onos-logs}
 LOGBASE=${ONOS_LOGBASE:-onos.${ONOS_HOST_NAME}}
 ONOS_LOG="${LOGDIR}/${LOGBASE}.log"
@@ -149,16 +163,16 @@
 }
 
 function rotate-log {
-    local logfile=$1
-    local nr_max=${2:-10}
-    if [ -f $logfile ]; then
-	for i in `seq $(expr $nr_max - 1) -1 1`; do
-	    if [ -f ${logfile}.${i} ]; then
-		mv -f ${logfile}.${i} ${logfile}.`expr $i + 1`
-	    fi
-	done
-	mv $logfile $logfile.1
-    fi
+  local logfile=$1
+  local nr_max=${2:-10}
+  if [ -f $logfile ]; then
+    for i in `seq $(expr $nr_max - 1) -1 1`; do
+      if [ -f ${logfile}.${i} ]; then
+        mv -f ${logfile}.${i} ${logfile}.`expr $i + 1`
+      fi
+    done
+    mv $logfile $logfile.1
+  fi
 }
 
 # kill-processes {module-name} {array of pids}
@@ -176,18 +190,29 @@
   done
 }
 
-function revert-conf {
+# revert-file {filename}
+# revert "filename" from "filename.bak" if "filename.tmp" exists.
+function revert-file {
+  local filename=$1
+  local temp="${filename}.tmp"
+  local backup="${filename}.bak"
+  
+  if [ -f "${temp}" ]; then
+    echo -n "reverting ${filename} ... "
+    mv ${backup} ${filename}
+    rm ${temp}
+    echo "DONE"
+  fi
+}
+
+# revert-confs [error message]
+function revert-confs {
   set -e
   
   echo -n "ERROR occurred ... "
-  local temp_zk="${ZK_CONF}.tmp"
   
-  if [ -f "${temp_zk}" ]; then
-    local zk_file=`basename ${ZK_CONF}`
-    echo -n "reverting ${zk_file} ... "
-    rm ${temp_zk}
-    mv ${ZK_CONF_BACKUP} ${ZK_CONF}
-  fi
+  revert-file `basename ${ZK_CONF}`
+  revert-file `basename ${HC_CONF}`
 
   echo "EXIT"
   
@@ -206,21 +231,18 @@
   # creation of zookeeper config
   
   local temp_zk="${ZK_CONF}.tmp"
+  if [ -f ${temp_zk} ]; then
+    rm ${temp_zk}
+  fi
   touch ${temp_zk}
-  
+
   if [ -f ${ZK_CONF} ]; then
-    mv ${ZK_CONF} ${ZK_CONF_BACKUP}
-    local backup_file=`basename ${ZK_CONF_BACKUP}`
-    echo -n "backup old file to ${backup_file} ... "
+    mv ${ZK_CONF} ${ZK_CONF}.bak
+    local filename=`basename ${ZK_CONF}`
+    echo -n "backup old file to ${filename}.bak ... "
   fi
   
-  local hosts=${ZK_HOSTS}
-  if [ -z "${hosts}" ]; then
-    # assume single-node mode
-    hosts=${ONOS_HOST_NAME}
-  fi
-  
-  hostarr=`echo ${hosts} | tr "," " "`
+  hostarr=`echo ${ZK_HOSTS} | tr "," " "`
   
   local i=1
   local myid=
@@ -234,7 +256,7 @@
   
   if [ -z "${myid}" ]; then
     local filename=`basename ${ONOS_CONF}`
-    revert-conf "[ERROR in ${filename}] zookeeper.hosts must have hostname \"${ONOS_HOST_NAME}\""
+    revert-confs "[ERROR in ${filename}] zookeeper.hosts must have hostname \"${ONOS_HOST_NAME}\""
   fi
   
   # TODO: Remove sudo.
@@ -264,51 +286,81 @@
   echo "DONE"
 }
 
+function create-hazelcast-conf {
+  echo -n "Creating ${HC_CONF} ... "
+  
+  local temp_hc="${HC_CONF}.tmp"
+  if [ -f ${temp_hc} ]; then
+    rm ${temp_hc}
+  fi
+  touch ${temp_hc}
+  
+  if [ -f ${HC_CONF} ]; then
+    mv ${HC_CONF} ${HC_CONF}.bak
+    local filename=`basename ${HC_CONF}`
+    echo -n "backup old file to ${filename}.bak ... "
+  fi
+  
+  # To keep indent of XML file, change IFS
+  local IFS=''
+  while read line; do
+    if [[ $line =~ __HC_NETWORK__ ]]; then
+      if [ ! -z "${HC_TCPIP_MEMBERS}" ]; then
+        # temporary change
+        IFS=' '
+        local memberarr=`echo ${HC_TCPIP_MEMBERS} | tr "," " "`
+        echo '<multicast enabled="false" />'
+        echo '<tcp-ip enabled="true">'
+        for member in ${memberarr}; do
+          echo "  <member>${member}</member>"
+        done
+        echo '</tcp-ip>'
+        IFS=''
+      else
+        echo '<multicast enabled="true">'
+        echo "  <multicast-group>${HC_MULTICAST_GROUP}</multicast-group>"
+        echo "  <multicast-port>${HC_MULTICAST_PORT}</multicast-port>"
+        echo '</multicast>'
+        echo '<tcp-ip enabled="false" />'
+      fi
+    else
+      echo "${line}"
+    fi
+  done < ${HC_CONF_TEMPLATE} > ${temp_hc}
+  
+  mv ${temp_hc} ${HC_CONF}
+  
+  echo "DONE"
+}
+
 function create-logback-conf {
   echo -n "Creating ${ONOS_LOGBACK} ... "
   
   # creation of logback config
   if [ -f $ONOS_LOGBACK ]; then
     local logback_file=`basename ${ONOS_LOGBACK}`
-    mv ${ONOS_LOGBACK} ${ONOS_LOGBACK_BACKUP}
-    local logback_back_file=`basename ${ONOS_LOGBACK_BACKUP}`
-    echo -n "backup old file to ${logback_back_file} ... "
+    mv ${ONOS_LOGBACK} ${ONOS_LOGBACK}.bak
+    local filename=`basename ${ONOS_LOGBACK}`
+    echo -n "backup old file to ${filename}.bak ... "
   fi
   sed -e "s|__FILENAME__|${ONOS_LOG}|" ${ONOS_LOGBACK_TEMPLATE} > ${ONOS_LOGBACK}
   
   echo "DONE"
 }
 
-function create-conf {
-  local key
-  local filename
+# create-conf-interactive {filename} {function to create conf}
+function create-conf-interactive {
+  local filepath=$1
+  local filename=`basename ${filepath}`
+  local func=$2
   
-  trap revert-conf ERR
-  
-  if [ -f ${ZK_CONF} -a "$1" != "-f" ]; then
+  if [ -f ${filepath} ]; then
     # confirmation to overwrite existing config file
-    echo -n "Overwriting ${ZK_CONF_FILE} [Y/n]? "
-    while [ 1 ]; do
-      read key
-      if [ -z "${key}" -o "${key}" == "Y" ]; then
-        create-zk-conf
-        break
-      elif [ "${key}" == "n" ]; then
-        break
-      fi
-      echo "[Y/n]?"
-    done
-  else
-    create-zk-conf
-  fi
-  
-  if [ -f ${ONOS_LOGBACK} -a "$1" != "-f" ]; then
-    filename=`basename ${ONOS_LOGBACK}`
     echo -n "Overwriting ${filename} [Y/n]? "
     while [ 1 ]; do
       read key
       if [ -z "${key}" -o "${key}" == "Y" ]; then
-        create-logback-conf
+        ${func}
         break
       elif [ "${key}" == "n" ]; then
         break
@@ -316,8 +368,26 @@
       echo "[Y/n]?"
     done
   else
-    create-logback-conf
+    ${func}
   fi
+}
+
+function create-confs {
+  local key
+  local filename
+  
+  trap revert-confs ERR
+
+  if [ "$1" == "-f" ]; then
+    create-zk-conf
+    create-hazelcast-conf
+    create-logback-conf
+  else
+    create-conf-interactive ${ZK_CONF} create-zk-conf
+    create-conf-interactive ${HC_CONF} create-hazelcast-conf
+    create-conf-interactive ${ONOS_LOGBACK} create-logback-conf
+  fi
+
   
   trap - ERR
 }
@@ -349,6 +419,7 @@
 function start-zk {
   echo -n "Starting Zookeeper ... "
   
+  export ZOO_LOG_DIR=${ZK_LOG_DIR}
   if [ -f "${ZK_CONF}" ]; then
     # Run Zookeeper with our configuration
     export ZOOCFG=${ZK_CONF_FILE}
@@ -404,47 +475,11 @@
 
 ### Functions related to RAMCloud coordinator
 function rc-coord-addr {
-  local coordproto=${RC_COORD_PROTOCOL}
-  local coordip=${RC_COORD_IP}
-  local coordport=${RC_COORD_PORT}
-
-  if [ -z "${coordproto}" ]; then
-    coordproto='fast+udp'
-  fi
-
-  if [ -z "${coordip}" ]; then
-    # assume single-node mode
-    coordip=${ONOS_HOST_IP}
-  fi
-  
-  if [ -z "${coordport}" ]; then
-    # assume default port
-    coordport=${RAMCLOUD_COORD_PORT}
-  fi
-  
-  echo "${coordproto}:host=${coordip},port=${coordport}"
+  echo "${RC_COORD_PROTOCOL}:host=${RC_COORD_IP},port=${RC_COORD_PORT}"
 }
 
 function rc-server-addr {
-  local serverproto=${RC_SERVER_PROTOCOL}
-  local serverip=${RC_SERVER_IP}
-  local serverport=${RC_SERVER_PORT}
-
-  if [ -z "${serverproto}" ]; then
-    serverproto='fast+udp'
-  fi
-
-  # Normally this parameter should be null
-  if [ -z "${serverip}" ]; then
-    serverip=${ONOS_HOST_IP}
-  fi
-  
-  if [ -z "${serverport}" ]; then
-    # assume default port
-    serverport=${RAMCLOUD_SERVER_PORT}
-  fi
-  
-  echo "${serverproto}:host=${serverip},port=${serverport}"
+  echo "${RC_SERVER_PROTOCOL}:host=${RC_SERVER_IP},port=${RC_SERVER_PORT}"
 }
 
 function rc-coord {
@@ -638,7 +673,7 @@
 ################## Main ####################
 case "$1" in
   setup)
-    create-conf $2
+    create-confs $2
     ;;
   start)
     mode_parameter=${ONOS_HOST_ROLE}