Fixed null providers & custom topo simulator to work across stop/starts.

Change-Id: Id8eeca73c8fc021c351d365ef009b3a95562763a
diff --git a/providers/null/src/main/java/org/onosproject/provider/nil/CustomTopologySimulator.java b/providers/null/src/main/java/org/onosproject/provider/nil/CustomTopologySimulator.java
index 52883a2..e7a5788 100644
--- a/providers/null/src/main/java/org/onosproject/provider/nil/CustomTopologySimulator.java
+++ b/providers/null/src/main/java/org/onosproject/provider/nil/CustomTopologySimulator.java
@@ -122,12 +122,12 @@
     protected void createHosts() {
     }
 
-    /**
-     * Resets the device and host ID seeds to the default values. That is, the
-     * next assigned values will start from 1 again.
-     */
-    public void resetIdSeeds() {
+    @Override
+    public void tearDownTopology() {
+        super.tearDownTopology();
         nextDeviceId = 0;
         nextHostId = 0;
+        nameToId.clear();
     }
+
 }
diff --git a/providers/null/src/main/java/org/onosproject/provider/nil/NullProviders.java b/providers/null/src/main/java/org/onosproject/provider/nil/NullProviders.java
index 7313f12..83471a5 100644
--- a/providers/null/src/main/java/org/onosproject/provider/nil/NullProviders.java
+++ b/providers/null/src/main/java/org/onosproject/provider/nil/NullProviders.java
@@ -385,6 +385,7 @@
             delay(500);
             simulator.tearDownTopology();
             simulator = null;
+
         }
     }
 
diff --git a/providers/null/src/main/java/org/onosproject/provider/nil/TopologySimulator.java b/providers/null/src/main/java/org/onosproject/provider/nil/TopologySimulator.java
index 2c79288..98efd7c 100644
--- a/providers/null/src/main/java/org/onosproject/provider/nil/TopologySimulator.java
+++ b/providers/null/src/main/java/org/onosproject/provider/nil/TopologySimulator.java
@@ -23,6 +23,7 @@
 import org.onlab.packet.VlanId;
 import org.onosproject.cluster.ClusterService;
 import org.onosproject.cluster.NodeId;
+import org.onosproject.mastership.MastershipAdminService;
 import org.onosproject.mastership.MastershipService;
 import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.Device;
@@ -58,6 +59,7 @@
 import static org.onlab.util.Tools.toHex;
 import static org.onosproject.net.HostId.hostId;
 import static org.onosproject.net.Link.Type.DIRECT;
+import static org.onosproject.net.MastershipRole.MASTER;
 import static org.onosproject.net.PortNumber.portNumber;
 import static org.onosproject.net.device.DeviceEvent.Type.*;
 import static org.onosproject.provider.nil.NullProviders.SCHEME;
@@ -78,6 +80,7 @@
 
     protected ClusterService clusterService;
     protected MastershipService mastershipService;
+    protected MastershipAdminService mastershipAdminService;
 
     protected DeviceAdminService deviceService;
     protected HostService hostService;
@@ -93,7 +96,7 @@
 
     protected final List<DeviceId> deviceIds = Lists.newArrayList();
 
-    private DeviceListener deviceEventCounter = new DeviceEventCounter();
+    private final DeviceListener deviceEventCounter = new DeviceEventCounter();
 
     /**
      * Initializes a new topology simulator with access to the specified service
@@ -118,6 +121,7 @@
 
         this.clusterService = directory.get(ClusterService.class);
         this.mastershipService = directory.get(MastershipService.class);
+        this.mastershipAdminService = directory.get(MastershipAdminService.class);
 
         this.deviceService = directory.get(DeviceAdminService.class);
         this.hostService = directory.get(HostService.class);
@@ -144,6 +148,7 @@
      * Sets up network topology simulation.
      */
     public void setUpTopology() {
+        deviceIds.clear();
         prepareForDeviceEvents(deviceCount);
         createDevices();
         waitForDeviceEvents();
@@ -204,6 +209,7 @@
                                              "ON.Lab", "0.1", "0.1", "1234",
                                              new ChassisId(chassisId));
         deviceIds.add(id);
+        mastershipAdminService.setRoleSync(localNode, id, MASTER);
         deviceProviderService.deviceConnected(id, desc);
         deviceProviderService.updatePorts(id, buildPorts(portCount));
     }
@@ -326,6 +332,8 @@
         prepareForDeviceEvents(deviceIds.size());
         deviceIds.forEach(deviceProviderService::deviceDisconnected);
         waitForDeviceEvents();
+        mastershipService.getDevicesOf(localNode).forEach(mastershipService::relinquishMastership);
+        deviceIds.forEach(mastershipService::relinquishMastership);
         deviceIds.clear();
     }
 
diff --git a/providers/null/src/main/java/org/onosproject/provider/nil/cli/CreateNullEntity.java b/providers/null/src/main/java/org/onosproject/provider/nil/cli/CreateNullEntity.java
index ed68dfc..368fa3d 100644
--- a/providers/null/src/main/java/org/onosproject/provider/nil/cli/CreateNullEntity.java
+++ b/providers/null/src/main/java/org/onosproject/provider/nil/cli/CreateNullEntity.java
@@ -17,6 +17,7 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Longs;
+import org.onlab.util.Tools;
 import org.onosproject.cli.AbstractShellCommand;
 import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.DeviceId;
@@ -96,11 +97,19 @@
         EdgePortService eps = get(EdgePortService.class);
         HostService hs = get(HostService.class);
 
-        List<ConnectPoint> points = ImmutableList
-                .sortedCopyOf((l, r) -> Longs.compare(l.port().toLong(), r.port().toLong()),
-                              eps.getEdgePoints(deviceId));
-        return points.stream()
-                .filter(p -> !Objects.equals(p, otherPoint) && hs.getConnectedHosts(p).isEmpty())
-                .findFirst().orElse(null);
+        // As there may be a slight delay in edge service getting updated, retry a few times
+        for (int i = 0; i < 3; i++) {
+            List<ConnectPoint> points = ImmutableList
+                    .sortedCopyOf((l, r) -> Longs.compare(l.port().toLong(), r.port().toLong()),
+                                  eps.getEdgePoints(deviceId));
+            ConnectPoint point = points.stream()
+                    .filter(p -> !Objects.equals(p, otherPoint) && hs.getConnectedHosts(p).isEmpty())
+                    .findFirst().orElse(null);
+            if (point != null) {
+                return point;
+            }
+            Tools.delay(100);
+        }
+        return null;
     }
 }
diff --git a/providers/null/src/main/java/org/onosproject/provider/nil/cli/NullControlCommand.java b/providers/null/src/main/java/org/onosproject/provider/nil/cli/NullControlCommand.java
index bfca14e..0b3fbe3 100644
--- a/providers/null/src/main/java/org/onosproject/provider/nil/cli/NullControlCommand.java
+++ b/providers/null/src/main/java/org/onosproject/provider/nil/cli/NullControlCommand.java
@@ -19,7 +19,6 @@
 import org.apache.karaf.shell.commands.Command;
 import org.onosproject.cfg.ComponentConfigService;
 import org.onosproject.cli.AbstractShellCommand;
-import org.onosproject.provider.nil.CustomTopologySimulator;
 import org.onosproject.provider.nil.NullProviders;
 import org.onosproject.provider.nil.TopologySimulator;
 
@@ -32,38 +31,32 @@
         description = "Starts or stops topology simulation")
 public class NullControlCommand extends AbstractShellCommand {
 
-    private static final String CUSTOM = "custom";
-
     @Argument(index = 0, name = "cmd", description = "Control command: start/stop",
-            required = true, multiValued = false)
+            required = true)
     String cmd = null;
 
     @Argument(index = 1, name = "topoShape",
             description = "Topology shape: e.g. configured, linear, reroute, " +
-                    "centipede, tree, spineleaf, mesh, fattree, custom",
-            required = false, multiValued = false)
+                    "centipede, tree, spineleaf, mesh, fattree, custom")
     String topoShape = null;
 
     @Override
     protected void execute() {
         ComponentConfigService service = get(ComponentConfigService.class);
+        // If there is an existing topology; make sure it's stopped before restarting
+        if (cmd.equals(START)) {
+            NullProviders npService = get(NullProviders.class);
+            TopologySimulator simulator = npService.currentSimulator();
+            if (simulator != null) {
+                simulator.tearDownTopology();
+            }
+        }
+
         if (topoShape != null) {
             service.setProperty(NullProviders.class.getName(), "topoShape", topoShape);
         }
         service.setProperty(NullProviders.class.getName(), "enabled",
                             cmd.equals(START) ? "true" : "false");
-
-        // If we are re-starting the "custom" topology, reset the counts
-        //  on the auto-assigned IDs for null-devices and null-hosts, so that
-        //  scripts can rely on consistent assignment of IDs to nodes.
-        if (CUSTOM.equals(topoShape) && START.equals(cmd)) {
-            NullProviders npService = get(NullProviders.class);
-            TopologySimulator simulator = npService.currentSimulator();
-            if (simulator instanceof CustomTopologySimulator) {
-                CustomTopologySimulator sim = (CustomTopologySimulator) simulator;
-                sim.resetIdSeeds();
-            }
-        }
     }
 
 }
diff --git a/tools/test/topos/cfab-null b/tools/test/topos/cfab-null
index 8993d2e..f8eb3f0 100755
--- a/tools/test/topos/cfab-null
+++ b/tools/test/topos/cfab-null
@@ -14,11 +14,16 @@
     echo "$@" >> $CMDS
 }
 
-sim "wipe-out please"
+# Generate the recipe using the following:
+# 2 spines, can potentially be few more
+# 12 leaves in total
+#     2 leaf pair
+#     8 non-paired
+# Host per leaf up to 1K
 
 spinePorts=48
 leafPorts=64
-accessPorts=1024
+accessPorts=128
 
 # Create spines
 for spine in {1..2}; do
@@ -56,29 +61,27 @@
     done
 
     # Create hosts for each single leaf
-    for host in {1..25}; do
+    for host in {1..50}; do
         sim "null-create-host Access-${access} 10.0.${access}.${host}"
     done
 done
 
 
-
-# make sure null providers are activated
+# make sure null providers are activated and any running simulation is stopped
 onos ${node} app activate org.onosproject.null
-
 sleep 2
+onos ${node} null-simulation stop
+
+# wait until the masterships clear-out across the cluster
+while onos ${node} masters | grep -qv " 0 devices"; do sleep 1; done
+
+# clean-up
+onos ${node} wipe-out please
+sleep 1
 
 # start custom simulation..
 onos ${node} null-simulation start custom
-
-# Generate the recipe using the following:
-# 2 spines, can potentially be few more
-# 12 leaves in total
-#     2 leaf pair
-#     8 non-paired
-# Host per leaf up to 1K
-
-sleep 5
+sleep 2
 
 # Add devices, links, and hosts
 cat $CMDS | onos ${node}