[ONOS-7599] Test application to test cluster stability after multiple SDNC node reboots

Change-Id: Iedb57017d9247d13569b2ccfea559d905286acf3
diff --git a/apps/test/cluster-ha/BUCK b/apps/test/cluster-ha/BUCK
new file mode 100644
index 0000000..feb9b6b
--- /dev/null
+++ b/apps/test/cluster-ha/BUCK
@@ -0,0 +1,16 @@
+COMPILE_DEPS = [
+    '//lib:CORE_DEPS',
+    '//lib:org.apache.karaf.shell.console',
+    '//cli:onos-cli',
+    '//core/store/serializers:onos-core-serializers',
+    '//lib:KRYO',
+]
+osgi_jar (
+    deps = COMPILE_DEPS,
+)
+onos_app (
+    title = 'Cluster HA Test',
+    category = 'Cluster Test Utility',
+    url = 'http://onosproject.org',
+    description = 'Test for ONOS Cluster HA',
+)
diff --git a/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/ClusterHATest.java b/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/ClusterHATest.java
new file mode 100644
index 0000000..0ccca0a
--- /dev/null
+++ b/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/ClusterHATest.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.onosproject.clusterha;
+
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.StorageException;
+import org.onosproject.store.service.StorageService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * Cluster HA Test.
+ */
+@Component(immediate = true)
+@Service(value = ClusterHATest.class)
+public class ClusterHATest {
+
+    protected final Logger log = LoggerFactory.getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected StorageService storageService;
+
+    private ApplicationId appId;
+    private List<ConsistentMap<String, String>> stringStoreList;
+    private int numStores = 10;
+
+    @Activate
+    protected void activate() {
+        appId = coreService.registerApplication("org.onosproject.cluster");
+        stringStoreList = new ArrayList<ConsistentMap<String, String>>();
+        for (int i = 0; i < numStores; i++) {
+            ConsistentMap<String, String> stringStore = storageService.<String, String>consistentMapBuilder()
+                    .withName("cluster-ha-store" + i)
+                    .withApplicationId(appId)
+                    .withSerializer(Serializer.using(new KryoNamespace.Builder()
+                                                             .register(KryoNamespaces.API)
+                                                             .register(String.class)
+                                                             .build("string")))
+                    .build();
+            stringStoreList.add(stringStore);
+        }
+        log.info("Started");
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        for (ConsistentMap<String, String> stringStore: stringStoreList) {
+            stringStore.clear();
+        }
+        log.info("Stopped");
+    }
+
+    public void addToStringStore(int key, String str) {
+        try {
+            for (ConsistentMap<String, String> stringStore: stringStoreList) {
+                stringStore.put(String.valueOf(key), str);
+            }
+        } catch (StorageException e) {
+            log.error("StringStore - put got exception {} , for the key {} string {}", e, key, str);
+        }
+    }
+
+    public void removeStringFromStore(int key) {
+        try {
+            for (ConsistentMap<String, String> stringStore: stringStoreList) {
+                stringStore.remove(String.valueOf(key));
+            }
+        } catch (StorageException e) {
+            log.error("StringStore - put got exception {} , for the key {}", e, key);
+        }
+    }
+}
\ No newline at end of file
diff --git a/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/cli/ClusterHATestCommand.java b/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/cli/ClusterHATestCommand.java
new file mode 100644
index 0000000..6422988
--- /dev/null
+++ b/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/cli/ClusterHATestCommand.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.clusterha.cli;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.clusterha.ClusterHATest;
+
+@Command(scope = "onos", name = "cluster-ha-test",
+        description = "test addition & deletion on consistent map")
+public class ClusterHATestCommand extends AbstractShellCommand {
+
+    @Argument(index = 0, name = "operation", description = "add/del", required = true)
+    private String operation = null;
+
+    @Argument(index = 1, name = "strStartIdx", description = "start index", required = true)
+    private String strStartIdx = null;
+
+    @Argument(index = 2, name = "strEndIdx", description = "end index", required = true)
+    private String strEndIdx = null;
+
+    @Argument(index = 3, name = "strInterOpDelay", description = "inter operation delay(ms)", required = true)
+    private String strInterOpDelay = null;
+
+    private static final String OP_ADD = "add";
+    private static final String OP_DEL = "del";
+
+    private static final int COUNT = 100;
+    private static final String STR = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+    private static final String CONSTANT = new String(new char[COUNT]).replace("\0", STR);
+
+    @Override
+    protected void execute() {
+        ClusterHATest service = get(ClusterHATest.class);
+        int iStartIdx = 0, iEndIdx = 0;
+        long lInterOpDelay = 0L;
+
+        try {
+            iStartIdx = Integer.parseInt(strStartIdx);
+            iEndIdx = Integer.parseInt(strEndIdx);
+            lInterOpDelay = Long.parseLong(strInterOpDelay);
+        } catch (Exception e) {
+            print(e.getMessage());
+            return;
+        }
+
+        if (!OP_ADD.equals(operation) && !OP_DEL.equals(operation)) {
+            print("invalid operation " + operation);
+            return;
+        }
+
+        print("cluster-ha-test " + operation + " " + iStartIdx + " " + iEndIdx + " " + lInterOpDelay);
+
+        if (OP_ADD.equals(operation)) {
+            for (int i = iStartIdx; i <= iEndIdx; i++) {
+                service.addToStringStore(i, CONSTANT);
+                try {
+                    Thread.sleep(lInterOpDelay);
+                } catch (Exception e) {
+                    return;
+                }
+            }
+
+        } else if (OP_DEL.equals(operation)) {
+            for (int i = iStartIdx; i <= iEndIdx; i++) {
+                service.removeStringFromStore(i);
+                try {
+                    Thread.sleep(lInterOpDelay);
+                } catch (Exception e) {
+
+                }
+            }
+        } else {
+            print("Invalid operation " + operation);
+        }
+    }
+}
\ No newline at end of file
diff --git a/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/cli/package-info.java b/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/cli/package-info.java
new file mode 100644
index 0000000..4c04e52
--- /dev/null
+++ b/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/cli/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *  Cluster HA Test CLI.
+ */
+
+package org.onosproject.clusterha.cli;
diff --git a/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/package-info.java b/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/package-info.java
new file mode 100644
index 0000000..c5e09a0
--- /dev/null
+++ b/apps/test/cluster-ha/src/main/java/org/onosproject/clusterha/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *  Cluster HA Test.
+ */
+package org.onosproject.clusterha;
\ No newline at end of file
diff --git a/apps/test/cluster-ha/src/main/resources/OSGI-INF/blueprint/shell-config.xml b/apps/test/cluster-ha/src/main/resources/OSGI-INF/blueprint/shell-config.xml
new file mode 100644
index 0000000..55cd87f
--- /dev/null
+++ b/apps/test/cluster-ha/src/main/resources/OSGI-INF/blueprint/shell-config.xml
@@ -0,0 +1,25 @@
+<!--
+  ~ Copyright 2018-present Open Networking Foundation
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0">
+
+    <command-bundle xmlns="http://karaf.apache.org/xmlns/shell/v1.1.0">
+        <command>
+            <action class="org.onosproject.clusterha.cli.ClusterHATestCommand"/>
+        </command>
+    </command-bundle>
+
+</blueprint>
diff --git a/modules.defs b/modules.defs
index 7184beb..5d422a2 100644
--- a/modules.defs
+++ b/modules.defs
@@ -179,6 +179,7 @@
     '//apps/reactive-routing:onos-apps-reactive-routing-oar',
     '//apps/roadm:onos-apps-roadm-oar',
     '//apps/sdnip:onos-apps-sdnip-oar',
+    '//apps/test/cluster-ha:onos-apps-test-cluster-ha-oar',
     '//apps/test/demo:onos-apps-test-demo-oar',
     '//apps/test/distributed-primitives:onos-apps-test-distributed-primitives-oar',
     '//apps/test/election:onos-apps-test-election-oar',