| Thomas Vachuska | 4519770 | 2018-05-16 11:52:03 -0700 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | #------------------------------------------------------------------------------- |
| 3 | # Utility to repair cell slave by destroying all cell-related containers and |
| 4 | # by cancelling all cell reservations hosted by that slave. Intended for use |
| 5 | # after the cell slave has been rebooted as a result of some H/W or OS issues. |
| 6 | #------------------------------------------------------------------------------- |
| 7 | |
| 8 | slave=$1 |
| 9 | |
| 10 | [ -z "$slave" -o "$slave" = "-h" -o "$slave" = "-?" ] \ |
| 11 | && echo "usage: $(basename $0) slaveIp" && exit 1 |
| 12 | |
| 13 | # Destroy all cell-related containers on the specified slave; do not touch |
| 14 | # the base-* template images though. |
| 15 | ssh sdn@$slave " |
| Thomas Vachuska | cdb0527 | 2018-08-15 12:43:41 -0700 | [diff] [blame] | 16 | containers=\$(sudo lxc-ls --fancy | grep -E '[a-z]*-[0-9cn]' | cut -d\ -f1) |
| Thomas Vachuska | 4519770 | 2018-05-16 11:52:03 -0700 | [diff] [blame] | 17 | for c in \$containers; do |
| 18 | sudo lxc-stop -n \$c |
| 19 | sudo lxc-destroy -n \$c |
| 20 | done |
| 21 | " |
| 22 | # Cancel all reservations hosted on the specified slave |
| 23 | # Also, kill any warden spawned processes |
| 24 | ssh sdn@$CELL_WARDEN " |
| 25 | cells=\$(grep -l $slave warden/cells/supported/* | xargs -n1 basename) |
| 26 | for c in \$cells; do |
| 27 | [ -f warden/cells/reserved/\$c ] && rm warden/cells/reserved/\$c |
| 28 | done |
| 29 | pids=\$(ps -ef | grep warden | grep -Ev 'grep|java' | cut -c10-15) |
| 30 | [ -n \"\$pids\" ] && kill -9 \$pids |
| 31 | " |