Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 1 | #!/bin/bash |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 2 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 3 | ### Env vars used by this script. (default value) ### |
| 4 | # $ONOS_HOME : path of root directory of ONOS repository (this script's dir) |
| 5 | # $ONOS_CONF_DIR : path of ONOS config directory (~/ONOS/conf) |
| 6 | # $ONOS_CONF : path of ONOS node config file (~/ONOS/conf/onos_node.conf) |
| 7 | # $ONOS_PROPS : path of ONOS properties file (~/ONOS/conf/onos.properties) |
| 8 | # $ONOS_LOGBACK : path of logback config file (~/ONOS/conf/logback.`hostname`.xml) |
| 9 | # $LOGDIR : path of log output directory (~/ONOS/onos-logs) |
| 10 | # $LOGBASE : base name of log output file (onos.`hostname`) |
| 11 | # $RAMCLOUD_HOME : path of root directory of RAMCloud repository (~/ramcloud) |
| 12 | # $RAMCLOUD_BRANCH : branch name of RAMCloud to use (master) |
| 13 | # $JVM_OPTS : JVM options ONOS starts with |
| 14 | ##################################################### |
| 15 | |
| 16 | |
| 17 | ### Variables read from ONOS config file ### |
| 18 | ONOS_HOME=${ONOS_HOME:-$(cd `dirname $0`; pwd)} |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 19 | ONOS_CONF_DIR=${ONOS_CONF_DIR:-${ONOS_HOME}/conf} |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 20 | ONOS_CONF=${ONOS_CONF:-${ONOS_CONF_DIR}/onos_node.conf} |
| 21 | |
| 22 | if [ ! -f ${ONOS_CONF} ]; then |
| 23 | echo "${ONOS_CONF} not found." |
| 24 | exit 1 |
| 25 | fi |
| 26 | ONOS_HOST_NAME=`grep ^host.name ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 27 | if [ -z "${ONOS_HOST_NAME}" ]; then |
| 28 | ONOS_HOST_NAME=`hostname` |
| 29 | fi |
| 30 | ONOS_HOST_IP=`grep ^host.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 31 | ONOS_HOST_ROLE=`grep ^host.role ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 32 | ONOS_HOST_BACKEND=`grep ^host.backend ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 33 | ZK_HOSTS=`grep ^zookeeper.hosts ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 34 | RC_COORD_PROTOCOL=`grep ^ramcloud.coordinator.protocol ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 35 | RC_COORD_IP=`grep ^ramcloud.coordinator.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 36 | RC_COORD_PORT=`grep ^ramcloud.coordinator.port ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 37 | RC_SERVER_PROTOCOL=`grep ^ramcloud.server.protocol ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 38 | RC_SERVER_IP=`grep ^ramcloud.server.ip ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 39 | RC_SERVER_PORT=`grep ^ramcloud.server.port ${ONOS_CONF} | cut -d "=" -f 2 | sed -e 's/^[ \t]*//'` |
| 40 | ############################################ |
| 41 | |
| 42 | |
| 43 | ############## Other variables ############# |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 44 | LOGDIR=${ONOS_LOGDIR:-${ONOS_HOME}/onos-logs} |
| 45 | |
| 46 | ZK_DIR=${HOME}/zookeeper-3.4.5 |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 47 | ZK_CONF_FILE=zoo.cfg |
| 48 | ZK_CONF=${ONOS_CONF_DIR}/${ZK_CONF_FILE} |
| 49 | ZK_CONF_BACKUP=${ZK_CONF}.bak |
| 50 | ZK_CONF_TEMPLATE=${ONOS_CONF_DIR}/zoo.cfg.template |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 51 | ZOO_LOG_DIR=${ONOS_HOME}/onos-logs |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 52 | ZK_LIB_DIR=/var/lib/zookeeper |
| 53 | ZK_MY_ID=${ZK_LIB_DIR}/myid |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 54 | |
| 55 | RAMCLOUD_DIR=${HOME}/ramcloud |
| 56 | RAMCLOUD_HOME=${RAMCLOUD_HOME:-~/ramcloud} |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 57 | RAMCLOUD_COORD_LOG=${LOGDIR}/ramcloud.coordinator.${ONOS_HOST_NAME}.log |
| 58 | RAMCLOUD_SERVER_LOG=${LOGDIR}/ramcloud.server.${ONOS_HOST_NAME}.log |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 59 | RAMCLOUD_BRANCH=${RAMCLOUD_BRANCH:-master} |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 60 | RAMCLOUD_COORD_PORT=12246 |
| 61 | RAMCLOUD_SERVER_PORT=12242 |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 62 | |
| 63 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ONOS_HOME}/lib:${RAMCLOUD_HOME}/obj.${RAMCLOUD_BRANCH} |
| 64 | |
| 65 | ## Because the script change dir to $ONOS_HOME, we can set ONOS_LOGBACK and LOGDIR relative to $ONOS_HOME |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 66 | ONOS_LOGBACK=${ONOS_LOGBACK:-${ONOS_CONF_DIR}/logback.${ONOS_HOST_NAME}.xml} |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 67 | ONOS_LOGBACK_BACKUP=${ONOS_LOGBACK}.bak |
| 68 | ONOS_LOGBACK_TEMPLATE=${ONOS_CONF_DIR}/logback.xml.template |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 69 | LOGDIR=${ONOS_LOGDIR:-${ONOS_HOME}/onos-logs} |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 70 | LOGBASE=${ONOS_LOGBASE:-onos.${ONOS_HOST_NAME}} |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 71 | ONOS_LOG="${LOGDIR}/${LOGBASE}.log" |
| 72 | PCAP_LOG="${LOGDIR}/${LOGBASE}.pcap" |
| 73 | LOGS="$ONOS_LOG $PCAP_LOG" |
| 74 | |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 75 | ONOS_PROPS=${ONOS_PROPS:-${ONOS_CONF_DIR}/onos.properties} |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 76 | JMX_PORT=${JMX_PORT:-7189} |
| 77 | |
| 78 | # Set JVM options |
| 79 | JVM_OPTS="${JVM_OPTS:-}" |
| 80 | ## If you want JaCoCo Code Coverage reports... uncomment line below |
| 81 | #JVM_OPTS="$JVM_OPTS -javaagent:${ONOS_HOME}/lib/jacocoagent.jar=dumponexit=true,output=file,destfile=${LOGDIR}/jacoco.exec" |
| 82 | JVM_OPTS="$JVM_OPTS -server -d64" |
| 83 | #JVM_OPTS="$JVM_OPTS -XX:+TieredCompilation -XX:InitialCodeCacheSize=512m -XX:ReservedCodeCacheSize=512m" |
| 84 | JVM_OPTS="$JVM_OPTS -Xmx4g -Xms4g -Xmn800m" |
| 85 | #JVM_OPTS="$JVM_OPTS -Xmx2g -Xms2g -Xmn800m" |
| 86 | #JVM_OPTS="$JVM_OPTS -Xmx1g -Xms1g -Xmn800m" |
| 87 | #JVM_OPTS="$JVM_OPTS -XX:+UseParallelGC -XX:+AggressiveOpts -XX:+UseFastAccessorMethods" |
| 88 | JVM_OPTS="$JVM_OPTS -XX:+UseConcMarkSweepGC -XX:+UseAdaptiveSizePolicy -XX:+AggressiveOpts -XX:+UseFastAccessorMethods" |
| 89 | JVM_OPTS="$JVM_OPTS -XX:MaxInlineSize=8192 -XX:FreqInlineSize=8192" |
| 90 | JVM_OPTS="$JVM_OPTS -XX:CompileThreshold=1500 -XX:PreBlockSpin=8" |
| 91 | JVM_OPTS="$JVM_OPTS -XX:OnError=crash-logger" ;# For dumping core |
| 92 | #JVM_OPTS="$JVM_OPTS -Dpython.security.respectJavaAccessibility=false" |
| 93 | JVM_OPTS="$JVM_OPTS -XX:CompileThreshold=1500 -XX:PreBlockSpin=8 \ |
| 94 | -XX:+UseThreadPriorities \ |
| 95 | -XX:ThreadPriorityPolicy=42 \ |
| 96 | -XX:+UseCompressedOops \ |
| 97 | -Dcom.sun.management.jmxremote.port=$JMX_PORT \ |
| 98 | -Dcom.sun.management.jmxremote.ssl=false \ |
| 99 | -Dbenchmark.measureBP=0 \ |
| 100 | -Dbenchmark.measureRc=0 \ |
| 101 | -Dbenchmark.measureONOS=0 \ |
| 102 | -Dcom.sun.management.jmxremote.authenticate=false" |
| 103 | JVM_OPTS="$JVM_OPTS -Dhazelcast.logging.type=slf4j" |
| 104 | |
| 105 | # Set ONOS core main class |
| 106 | MAIN_CLASS="net.onrc.onos.ofcontroller.core.Main" |
| 107 | |
| 108 | MVN=${MVN:-mvn -o} |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 109 | ############################################ |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 110 | |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 111 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 112 | ############# Common functions ############# |
| 113 | function print_usage { |
| 114 | local filename=`basename ${ONOS_CONF}` |
| 115 | local usage="Usage: setup/start/stop ONOS on this server. |
| 116 | \$ $0 setup [-f] |
| 117 | Set up ONOS node using ${filename}. |
| 118 | - generate and replace config file of ZooKeeper. |
| 119 | - create myid in ZooKeeper datadir. |
| 120 | - generate and replace logback.${ONOS_HOST_NAME}.xml |
| 121 | If -f option is used, all existing files will be overwritten without confirmation. |
| 122 | \$ $0 start [single-node|coord-node|server-node|coord-and-server-node] |
| 123 | Start ONOS node with specific RAMCloud entities |
| 124 | - single-node: start ONOS with stand-alone RAMCloud |
| 125 | - coord-node : start ONOS with RAMCloud coordinator |
| 126 | - server-node: start ONOS with RAMCloud server |
| 127 | - coord-and-server-node: start ONOS with RAMCloud coordinator and server |
| 128 | * Default behavior can be defined by ${filename} |
| 129 | \$ $0 stop |
| 130 | Stop all ONOS-related processes |
| 131 | \$ $0 restart |
| 132 | Stop and start currently running ONOS-related processes |
| 133 | \$$0 status |
| 134 | Show status of ONOS-related processes |
| 135 | \$ $0 {zk|rc-coord|rc-server|core} {start|stop|restart|status} |
| 136 | Control specific ONOS-related process" |
| 137 | |
| 138 | echo "${usage}" |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 139 | } |
| 140 | |
| 141 | function rotate-log { |
| 142 | local logfile=$1 |
| 143 | local nr_max=${2:-10} |
| 144 | if [ -f $logfile ]; then |
| 145 | for i in `seq $(expr $nr_max - 1) -1 1`; do |
| 146 | if [ -f ${logfile}.${i} ]; then |
| 147 | mv -f ${logfile}.${i} ${logfile}.`expr $i + 1` |
| 148 | fi |
| 149 | done |
| 150 | mv $logfile $logfile.1 |
| 151 | fi |
| 152 | } |
| 153 | |
| 154 | # kill-processes {module-name} {array of pids} |
| 155 | function kill-processes { |
| 156 | # Kill the existing processes |
| 157 | local pids=$2 |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 158 | if [ ! -z "$pids" ]; then |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 159 | echo -n "Stopping $1 ... " |
| 160 | fi |
| 161 | for p in ${pids}; do |
| 162 | if [ x$p != "x" ]; then |
| 163 | kill -KILL $p |
| 164 | echo "Killed existing process (pid: $p)" |
| 165 | fi |
| 166 | done |
| 167 | } |
| 168 | |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 169 | function revert-conf { |
| 170 | set -e |
| 171 | |
| 172 | echo -n "ERROR occurred ... " |
| 173 | local temp_zk="${ZK_CONF}.tmp" |
| 174 | |
| 175 | if [ -f "${temp_zk}" ]; then |
| 176 | local zk_file=`basename ${ZK_CONF}` |
| 177 | echo -n "reverting ${zk_file} ... " |
| 178 | rm ${temp_zk} |
| 179 | mv ${ZK_CONF_BACKUP} ${ZK_CONF} |
| 180 | fi |
| 181 | |
| 182 | echo "EXIT" |
| 183 | |
| 184 | if [ ! -z "$1" ]; then |
| 185 | echo $1 |
| 186 | fi |
| 187 | |
| 188 | set +e |
| 189 | |
| 190 | exit 1 |
| 191 | } |
| 192 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 193 | function create-zk-conf { |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 194 | echo -n "Creating ${ZK_CONF} ... " |
| 195 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 196 | # creation of zookeeper config |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 197 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 198 | local temp_zk="${ZK_CONF}.tmp" |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 199 | touch ${temp_zk} |
| 200 | |
| 201 | if [ -f ${ZK_CONF} ]; then |
| 202 | mv ${ZK_CONF} ${ZK_CONF_BACKUP} |
| 203 | local backup_file=`basename ${ZK_CONF_BACKUP}` |
| 204 | echo -n "backup old file to ${backup_file} ... " |
| 205 | fi |
| 206 | |
| 207 | local hosts=${ZK_HOSTS} |
| 208 | if [ -z "${hosts}" ]; then |
| 209 | # assume single-node mode |
| 210 | hosts=${ONOS_HOST_NAME} |
| 211 | fi |
| 212 | |
| 213 | hostarr=`echo ${hosts} | tr "," " "` |
| 214 | |
| 215 | local i=1 |
| 216 | local myid= |
| 217 | for host in ${hostarr}; do |
| 218 | if [ ${host} = ${ONOS_HOST_NAME} ]; then |
| 219 | myid=$i |
| 220 | break |
| 221 | fi |
| 222 | i=`expr $i + 1` |
| 223 | done |
| 224 | |
| 225 | if [ -z "${myid}" ]; then |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 226 | local filename=`basename ${ONOS_CONF}` |
| 227 | revert-conf "[ERROR in ${filename}] zookeeper.hosts must have hostname \"${ONOS_HOST_NAME}\"" |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 228 | fi |
| 229 | |
| 230 | # TODO: Remove sudo. |
| 231 | # This is temporary code for the sake of compatibility with old code (which creates myid with root user). |
| 232 | sudo mv ${ZK_MY_ID} ${ZK_MY_ID}.old |
| 233 | echo ${myid} > ${ZK_MY_ID} |
| 234 | |
| 235 | echo -n "myid is assigned to ${myid} ... " |
| 236 | |
| 237 | while read line; do |
| 238 | if [[ $line =~ ^__HOSTS__$ ]]; then |
| 239 | i=1 |
| 240 | for host in ${hostarr}; do |
| 241 | # TODO: ports might be configurable |
| 242 | local hostline="server.${i}=${host}:2888:3888" |
| 243 | echo $hostline >> "${temp_zk}" |
| 244 | i=`expr $i + 1` |
| 245 | done |
| 246 | elif [[ $line =~ __DATADIR__ ]]; then |
| 247 | echo $line | sed -e "s|__DATADIR__|${ZK_LIB_DIR}|" >> ${temp_zk} |
| 248 | else |
| 249 | echo $line >> ${temp_zk} |
| 250 | fi |
| 251 | done < ${ZK_CONF_TEMPLATE} |
| 252 | mv ${temp_zk} ${ZK_CONF} |
| 253 | |
| 254 | echo "DONE" |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 255 | } |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 256 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 257 | function create-logback-conf { |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 258 | echo -n "Creating ${ONOS_LOGBACK} ... " |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 259 | |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 260 | # creation of logback config |
| 261 | if [ -f $ONOS_LOGBACK ]; then |
| 262 | local logback_file=`basename ${ONOS_LOGBACK}` |
| 263 | mv ${ONOS_LOGBACK} ${ONOS_LOGBACK_BACKUP} |
| 264 | local logback_back_file=`basename ${ONOS_LOGBACK_BACKUP}` |
| 265 | echo -n "backup old file to ${logback_back_file} ... " |
| 266 | fi |
| 267 | sed -e "s|__FILENAME__|${ONOS_LOG}|" ${ONOS_LOGBACK_TEMPLATE} > ${ONOS_LOGBACK} |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 268 | |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 269 | echo "DONE" |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 270 | } |
| 271 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 272 | function create-conf { |
| 273 | local key |
| 274 | local filename |
| 275 | |
| 276 | trap revert-conf ERR |
| 277 | |
| 278 | if [ -f ${ZK_CONF} -a "$1" != "-f" ]; then |
| 279 | # confirmation to overwrite existing config file |
| 280 | echo -n "Overwriting ${ZK_CONF_FILE} [Y/n]? " |
| 281 | while [ 1 ]; do |
| 282 | read key |
| 283 | if [ -z "${key}" -o "${key}" == "Y" ]; then |
| 284 | create-zk-conf |
| 285 | break |
| 286 | elif [ "${key}" == "n" ]; then |
| 287 | break |
| 288 | fi |
| 289 | echo "[Y/n]?" |
| 290 | done |
| 291 | else |
| 292 | create-zk-conf |
| 293 | fi |
| 294 | |
| 295 | if [ -f ${ONOS_LOGBACK} -a "$1" != "-f" ]; then |
| 296 | filename=`basename ${ONOS_LOGBACK}` |
| 297 | echo -n "Overwriting ${filename} [Y/n]? " |
| 298 | while [ 1 ]; do |
| 299 | read key |
| 300 | if [ -z "${key}" -o "${key}" == "Y" ]; then |
| 301 | create-logback-conf |
| 302 | break |
| 303 | elif [ "${key}" == "n" ]; then |
| 304 | break |
| 305 | fi |
| 306 | echo "[Y/n]?" |
| 307 | done |
| 308 | else |
| 309 | create-logback-conf |
| 310 | fi |
| 311 | |
| 312 | trap - ERR |
| 313 | } |
| 314 | ############################################ |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 315 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 316 | |
| 317 | ###### Functions related to ZooKeeper ###### |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 318 | function zk { |
| 319 | case "$1" in |
| 320 | start) |
| 321 | start-zk |
| 322 | ;; |
| 323 | stop) |
| 324 | stop-zk |
| 325 | ;; |
| 326 | stat*) # <- status |
| 327 | status-zk |
| 328 | ;; |
| 329 | re*) # <- restart |
| 330 | stop-zk |
| 331 | start-zk |
| 332 | ;; |
| 333 | *) |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 334 | print_usage |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 335 | exit 1 |
| 336 | esac |
| 337 | } |
| 338 | |
| 339 | function start-zk { |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 340 | echo -n "Starting Zookeeper ... " |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 341 | |
| 342 | if [ -f "${ZK_CONF}" ]; then |
| 343 | # Run Zookeeper with our configuration |
| 344 | export ZOOCFG=${ZK_CONF_FILE} |
| 345 | export ZOOCFGDIR=${ONOS_CONF_DIR} |
| 346 | fi |
| 347 | |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 348 | $ZK_DIR/bin/zkServer.sh start |
| 349 | } |
| 350 | |
| 351 | function stop-zk { |
| 352 | kill-processes "ZooKeeper" `jps -l | grep org.apache.zookeeper.server | awk '{print $1}'` |
| 353 | } |
| 354 | |
| 355 | function status-zk { |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 356 | if [ -f ${ZK_CONF} ]; then |
| 357 | export ZOOCFG=${ZK_CONF_FILE} |
| 358 | export ZOOCFGDIR=${ONOS_CONF_DIR} |
| 359 | fi |
| 360 | |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 361 | $ZK_DIR/bin/zkServer.sh status |
| 362 | } |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 363 | ############################################ |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 364 | |
| 365 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 366 | ####### Functions related to RAMCloud ###### |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 367 | function start-backend { |
| 368 | if [ "${ONOS_HOST_BACKEND}" = "ramcloud" ]; then |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 369 | if [ $1 == "coord" ]; then |
| 370 | rc-coord startifdown |
| 371 | elif [ $1 == "server" ]; then |
| 372 | rc-server startifdown |
| 373 | fi |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 374 | fi |
| 375 | } |
| 376 | |
| 377 | function stop-backend { |
| 378 | rcsn=`pgrep -f obj.${RAMCLOUD_BRANCH}/server | wc -l` |
| 379 | if [ $rcsn != 0 ]; then |
| 380 | rc-server stop |
| 381 | fi |
| 382 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 383 | rccn=`pgrep -f obj.${RAMCLOUD_BRANCH}/coordinator | wc -l` |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 384 | if [ $rccn != 0 ]; then |
| 385 | rc-coord stop |
| 386 | fi |
| 387 | } |
| 388 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 389 | function deldb { |
| 390 | # TODO: implement |
| 391 | return |
| 392 | } |
| 393 | |
| 394 | |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 395 | ### Functions related to RAMCloud coordinator |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 396 | function rc-coord-addr { |
| 397 | local coordproto=${RC_COORD_PROTOCOL} |
| 398 | local coordip=${RC_COORD_IP} |
| 399 | local coordport=${RC_COORD_PORT} |
| 400 | |
| 401 | if [ -z "${coordproto}" ]; then |
| 402 | coordproto='fast+udp' |
| 403 | fi |
| 404 | |
| 405 | if [ -z "${coordip}" ]; then |
| 406 | # assume single-node mode |
| 407 | coordip=${ONOS_HOST_IP} |
| 408 | fi |
| 409 | |
| 410 | if [ -z "${coordport}" ]; then |
| 411 | # assume default port |
| 412 | coordport=${RAMCLOUD_COORD_PORT} |
| 413 | fi |
| 414 | |
| 415 | echo "${coordproto}:host=${coordip},port=${coordport}" |
| 416 | } |
| 417 | |
| 418 | function rc-server-addr { |
| 419 | local serverproto=${RC_SERVER_PROTOCOL} |
| 420 | local serverip=${RC_SERVER_IP} |
| 421 | local serverport=${RC_SERVER_PORT} |
| 422 | |
| 423 | if [ -z "${serverproto}" ]; then |
| 424 | serverproto='fast+udp' |
| 425 | fi |
| 426 | |
| 427 | # Normally this parameter should be null |
| 428 | if [ -z "${serverip}" ]; then |
| 429 | serverip=${ONOS_HOST_IP} |
| 430 | fi |
| 431 | |
| 432 | if [ -z "${serverport}" ]; then |
| 433 | # assume default port |
| 434 | serverport=${RAMCLOUD_SERVER_PORT} |
| 435 | fi |
| 436 | |
| 437 | echo "${serverproto}:host=${serverip},port=${serverport}" |
| 438 | } |
| 439 | |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 440 | function rc-coord { |
| 441 | case "$1" in |
| 442 | start) |
| 443 | deldb |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 444 | stop-coord |
| 445 | start-coord |
| 446 | ;; |
| 447 | startifdown) |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 448 | local n=`pgrep -f obj.${RAMCLOUD_BRANCH}/coordinator | wc -l` |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 449 | if [ $n == 0 ]; then |
| 450 | start-coord |
| 451 | else |
| 452 | echo "$n instance of RAMCloud coordinator running" |
| 453 | fi |
| 454 | ;; |
| 455 | stop) |
| 456 | stop-coord |
| 457 | ;; |
| 458 | stat*) # <- status |
| 459 | local n=`pgrep -f obj.${RAMCLOUD_BRANCH}/coordinator | wc -l` |
| 460 | echo "$n RAMCloud coordinator running" |
| 461 | ;; |
| 462 | *) |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 463 | print_usage |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 464 | exit 1 |
| 465 | esac |
| 466 | } |
| 467 | |
| 468 | function start-coord { |
| 469 | if [ ! -d ${LOGDIR} ]; then |
| 470 | mkdir -p ${LOGDIR} |
| 471 | fi |
| 472 | if [ -f $RAMCLOUD_COORD_LOG ]; then |
| 473 | rotate-log $RAMCLOUD_COORD_LOG |
| 474 | fi |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 475 | |
| 476 | local coord_addr=`rc-coord-addr` |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 477 | |
| 478 | # Run ramcloud |
| 479 | echo -n "Starting RAMCloud coordinator ... " |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 480 | $RAMCLOUD_DIR/obj.${RAMCLOUD_BRANCH}/coordinator -L ${coord_addr} > $RAMCLOUD_COORD_LOG 2>&1 & |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 481 | echo "STARTED" |
| 482 | } |
| 483 | |
| 484 | |
| 485 | function stop-coord { |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 486 | kill-processes "RAMCloud coordinator" `pgrep -f obj.${RAMCLOUD_BRANCH}/coordinator` |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 487 | } |
| 488 | |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 489 | ### Functions related to RAMCloud server |
| 490 | function rc-server { |
| 491 | case "$1" in |
| 492 | start) |
| 493 | deldb |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 494 | stop-server |
| 495 | start-server |
| 496 | ;; |
| 497 | startifdown) |
| 498 | local n=`pgrep -f obj.${RAMCLOUD_BRANCH}/server | wc -l` |
| 499 | if [ $n == 0 ]; then |
| 500 | start-server |
| 501 | else |
| 502 | echo "$n instance of RAMCloud server running" |
| 503 | fi |
| 504 | ;; |
| 505 | stop) |
| 506 | stop-server |
| 507 | ;; |
| 508 | # deldb) |
| 509 | # deldb |
| 510 | # ;; |
| 511 | stat*) # <- status |
| 512 | n=`pgrep -f obj.${RAMCLOUD_BRANCH}/server | wc -l` |
| 513 | echo "$n RAMCloud server running" |
| 514 | ;; |
| 515 | *) |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 516 | print_usage |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 517 | exit 1 |
| 518 | esac |
| 519 | } |
| 520 | |
| 521 | function start-server { |
| 522 | if [ ! -d ${LOGDIR} ]; then |
| 523 | mkdir -p ${LOGDIR} |
| 524 | fi |
| 525 | if [ -f $RAMCLOUD_SERVER_LOG ]; then |
| 526 | rotate-log $RAMCLOUD_SERVER_LOG |
| 527 | fi |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 528 | |
| 529 | local coord_addr=`rc-coord-addr` |
| 530 | local server_addr=`rc-server-addr` |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 531 | |
| 532 | # Run ramcloud |
| 533 | echo -n "Starting RAMCloud server ... " |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 534 | ${RAMCLOUD_DIR}/obj.${RAMCLOUD_BRANCH}/server -M -L ${server_addr} -C ${coord_addr} --masterServiceThreads 1 --logCleanerThreads 1 --detectFailures 0 > $RAMCLOUD_SERVER_LOG 2>&1 & |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 535 | echo "STARTED" |
| 536 | } |
| 537 | |
| 538 | function stop-server { |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 539 | kill-processes "RAMCloud server" `pgrep -f obj.${RAMCLOUD_BRANCH}/server` |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 540 | } |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 541 | ############################################ |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 542 | |
| 543 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 544 | ## Functions related to ONOS core process ## |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 545 | function onos { |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 546 | CPFILE=${ONOS_HOME}/.javacp.${ONOS_HOST_NAME} |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 547 | if [ ! -f ${CPFILE} ]; then |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 548 | echo "ONOS core needs to be built" |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 549 | ${MVN} -f ${ONOS_HOME}/pom.xml compile |
| 550 | fi |
| 551 | JAVA_CP=`cat ${CPFILE}` |
| 552 | JAVA_CP="${JAVA_CP}:${ONOS_HOME}/target/classes" |
| 553 | |
| 554 | case "$1" in |
| 555 | start) |
| 556 | stop-onos |
| 557 | start-onos |
| 558 | ;; |
| 559 | startnokill) |
| 560 | start-onos |
| 561 | ;; |
| 562 | startifdown) |
| 563 | n=`jps -l | grep "${MAIN_CLASS}" | wc -l` |
| 564 | if [ $n == 0 ]; then |
| 565 | start-onos |
| 566 | else |
| 567 | echo "$n instance of onos running" |
| 568 | fi |
| 569 | ;; |
| 570 | stop) |
| 571 | stop-onos |
| 572 | ;; |
| 573 | stat*) # <- status |
| 574 | n=`jps -l | grep "${MAIN_CLASS}" | wc -l` |
| 575 | echo "$n instance of onos running" |
| 576 | ;; |
| 577 | *) |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 578 | print_usage |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 579 | exit 1 |
| 580 | esac |
| 581 | } |
| 582 | |
| 583 | function start-onos { |
| 584 | if [ ! -d ${LOGDIR} ]; then |
| 585 | mkdir -p ${LOGDIR} |
| 586 | fi |
| 587 | # Backup log files |
| 588 | for log in ${LOGS}; do |
| 589 | if [ -f ${log} ]; then |
| 590 | rotate-log ${log} |
| 591 | fi |
| 592 | done |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 593 | |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 594 | if [ ! -f ${ONOS_LOGBACK} ]; then |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 595 | echo "[WARNING] ${ONOS_LOGBACK} not found." |
| 596 | echo " Run \"\$ $0 setup\" to create." |
| 597 | exit 1 |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 598 | fi |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 599 | |
| 600 | JVM_OPTS="${JVM_OPTS} -Dnet.onrc.onos.datastore.backend=${ONOS_HOST_BACKEND}" |
| 601 | |
| 602 | # Run ONOS |
| 603 | |
| 604 | echo -n "Starting ONOS controller ..." |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 605 | java ${JVM_OPTS} -Dlogback.configurationFile=${ONOS_LOGBACK} -cp ${JAVA_CP} ${MAIN_CLASS} -cf ${ONOS_PROPS} > ${LOGDIR}/${LOGBASE}.stdout 2>${LOGDIR}/${LOGBASE}.stderr & |
| 606 | |
| 607 | # We need to wait a bit to find out whether starting the ONOS process succeeded |
| 608 | sleep 1 |
| 609 | |
| 610 | n=`jps -l |grep "${MAIN_CLASS}" | wc -l` |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 611 | if [ $n -ge 1 ]; then |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 612 | echo " STARTED" |
| 613 | else |
| 614 | echo " FAILED" |
| 615 | fi |
| 616 | |
| 617 | # echo "java ${JVM_OPTS} -Dlogback.configurationFile=${ONOS_LOGBACK} -jar ${ONOS_JAR} -cf ./onos.properties > /dev/null 2>&1 &" |
| 618 | # sudo -b /usr/sbin/tcpdump -n -i eth0 -s0 -w ${PCAP_LOG} 'tcp port 6633' > /dev/null 2>&1 |
| 619 | } |
| 620 | |
| 621 | function stop-onos { |
| 622 | kill-processes "ONOS controller" `jps -l | grep ${MAIN_CLASS} | awk '{print $1}'` |
| 623 | # kill-processes "tcpdump" `ps -edalf |grep tcpdump |grep ${PCAP_LOG} | awk '{print $4}'` |
| 624 | } |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 625 | ############################################ |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 626 | |
| 627 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 628 | ################## Main #################### |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 629 | case "$1" in |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 630 | setup) |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 631 | create-conf $2 |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 632 | ;; |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 633 | start) |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 634 | mode_parameter=${ONOS_HOST_ROLE} |
| 635 | if [ ! -z "$2" ]; then |
| 636 | mode_parameter=$2 |
| 637 | fi |
| 638 | |
| 639 | case "${mode_parameter}" in |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 640 | single-node) |
| 641 | zk start |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 642 | start-backend coord |
| 643 | start-backend server |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 644 | onos startifdown |
| 645 | ;; |
| 646 | coord-node) |
| 647 | zk start |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 648 | start-backend coord |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 649 | onos startifdown |
| 650 | ;; |
| 651 | server-node) |
| 652 | zk start |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 653 | start-backend server |
| 654 | onos startifdown |
| 655 | ;; |
| 656 | coord-and-server-node) |
| 657 | zk start |
| 658 | start-backend coord |
| 659 | start-backend server |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 660 | onos startifdown |
| 661 | ;; |
| 662 | *) |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 663 | print_usage |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 664 | ;; |
| 665 | esac |
| 666 | echo |
| 667 | ;; |
| 668 | stop) |
| 669 | on=`jps -l | grep "${MAIN_CLASS}" | wc -l` |
| 670 | if [ $on != 0 ]; then |
| 671 | onos stop |
| 672 | fi |
| 673 | |
Naoki Shiota | 9df15d3 | 2014-03-27 14:26:20 -0700 | [diff] [blame] | 674 | stop-backend |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 675 | |
| 676 | zkn=`jps -l | grep org.apache.zookeeper.server | wc -l` |
| 677 | if [ $zkn != 0 ]; then |
| 678 | zk stop |
| 679 | fi |
| 680 | echo |
| 681 | ;; |
| 682 | restart) |
| 683 | on=`jps -l | grep "${MAIN_CLASS}" | wc -l` |
| 684 | if [ $on != 0 ]; then |
| 685 | onos stop |
| 686 | fi |
| 687 | |
| 688 | rcsn=`pgrep -f obj.${RAMCLOUD_BRANCH}/server | wc -l` |
| 689 | if [ $rcsn != 0 ]; then |
| 690 | rc-server stop |
| 691 | fi |
| 692 | |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 693 | rccn=`pgrep -f obj.${RAMCLOUD_BRANCH}/coordinator | wc -l` |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 694 | if [ $rccn != 0 ]; then |
| 695 | rc-coord stop |
| 696 | fi |
| 697 | |
| 698 | zkn=`jps -l | grep org.apache.zookeeper.server | wc -l` |
| 699 | if [ $zkn != 0 ]; then |
| 700 | zk restart |
| 701 | fi |
| 702 | |
| 703 | if [ $rccn != 0 ]; then |
| 704 | rc-coord startifdown |
| 705 | fi |
| 706 | |
| 707 | if [ $rcsn != 0 ]; then |
| 708 | rc-server startifdown |
| 709 | fi |
| 710 | |
| 711 | if [ $on != 0 ]; then |
| 712 | onos startifdown |
| 713 | fi |
| 714 | echo |
| 715 | ;; |
| 716 | stat*) # <- status |
| 717 | echo '[ZooKeeper]' |
| 718 | zk status |
| 719 | echo |
| 720 | echo '[RAMCloud coordinator]' |
| 721 | rc-coord status |
| 722 | echo |
| 723 | echo '[RAMCloud server]' |
| 724 | rc-server status |
| 725 | echo |
| 726 | echo '[ONOS core]' |
| 727 | onos status |
| 728 | echo |
| 729 | ;; |
| 730 | zk) |
| 731 | zk $2 |
| 732 | ;; |
| 733 | rc-c*) # <- rc-coordinator |
| 734 | rc-coord $2 |
| 735 | ;; |
| 736 | rc-s*) # <- rc-server |
| 737 | rc-server $2 |
| 738 | ;; |
| 739 | core) |
| 740 | onos $2 |
| 741 | ;; |
| 742 | *) |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 743 | print_usage |
Naoki Shiota | 4e46318 | 2014-03-21 15:13:24 -0700 | [diff] [blame] | 744 | exit 1 |
| 745 | esac |
Naoki Shiota | 590c18d | 2014-03-31 17:52:59 -0700 | [diff] [blame] | 746 | |