tools/test/scenarios/ha-cluster-restart.xml

<!--
  ~ Copyright 2016-present Open Networking Laboratory
  ~
  ~ Licensed under the Apache License, Version 2.0 (the "License");
  ~ you may not use this file except in compliance with the License.
  ~ You may obtain a copy of the License at
  ~
  ~     http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  -->

<scenario name="ha-cluster-restart"
          description="ONOS cluster restart and recovery">
    <group name="Cluster-Restart" if="${OC2}">
        <group name="Fail">
            <parallel var="${OC#}">
                <step name="Node-Stop-${#}" exec="onos-service ${OC#} stop"/>
                <step name="Wait-for-Stop-${#}" requires="~Node-Stop-${#}"
                      exec="onos-wait-for-stop ${OC#}"/>
            </parallel>
        </group>

        <step name="Breathe" exec="sleep 5" requires="Fail"/>

        <group name="Recover" requires="~Breathe,Fail">
            <parallel var="${OC#}">
                <step name="Node-Start-${#}" exec="onos-service ${OC#} start"/>
                <step name="Wait-for-Start-${#}" requires="~Node-Start-${#}"
                      exec="onos-wait-for-start ${OC#}"/>
            </parallel>
        </group>

        <group name="Validate-Recovery" requires="Recover">
            <parallel var="${OC#}">
                <step name="Check-Nodes-${#}"
                      exec="onos-check-nodes ${OC#}"/>

                <step name="Check-Components-${#}" requires="~Check-Nodes-${#}"
                      exec="onos-check-components ${OC#}"/>

                <step name="Check-Logs-${#}" requires="~Check-Nodes-${#}"
                      exec="onos-check-logs --ignore-store-exceptions ${OC#}"/>
                <step name="Check-Apps-${#}" requires="~Check-Nodes-${#}"
                      exec="onos-check-apps ${OC#} ${ONOS_APPS} includes"/>

                <step name="Check-Summary-${#}" requires="~Check-Nodes-${#},~Check-Apps-${#}"
                      exec="onos-check-summary ${OC#} [0-9]* ${OTD} ${OTL} ${OTH}"/>
            </parallel>
        </group>

        <step name="Balance-Masters-Again" requires="Validate-Recovery"
              exec="onos ${OC1} balance-masters"/>
    </group>

    <group name="Cluster-Restart-NA" unless="${OC2}">
        <step name="No-Op" exec="echo Scenario not supported for this cell size"/>
    </group>

</scenario>