diff --git a/apps/t3/app/BUCK b/apps/t3/app/BUCK
new file mode 100644
index 0000000..55092ea
--- /dev/null
+++ b/apps/t3/app/BUCK
@@ -0,0 +1,26 @@
+COMPILE_DEPS = [
+    '//lib:CORE_DEPS',
+    '//lib:KRYO',
+    '//core/store/serializers:onos-core-serializers',
+    '//core/store/primitives:onos-core-primitives',
+    '//lib:JACKSON',
+    '//lib:jackson-databind',
+    '//core/api:onos-api',
+    '//lib:org.apache.karaf.shell.console',
+    '//cli:onos-cli',
+    '//drivers/default:onos-drivers-default',
+    '//apps/segmentrouting/app:onos-apps-segmentrouting-app',
+    '//apps/route-service/api:onos-apps-route-service-api',
+    '//apps/mcast/api:onos-apps-mcast-api',
+]
+
+TEST_DEPS = [
+    '//lib:TEST_ADAPTERS',
+    '//utils/misc:onlab-misc',
+    '//apps/route-service/api:onos-apps-route-service-api-tests',
+]
+
+osgi_jar_with_tests (
+    deps = COMPILE_DEPS,
+    test_deps = TEST_DEPS,
+)
diff --git a/apps/t3/app/pom.xml b/apps/t3/app/pom.xml
new file mode 100644
index 0000000..592a6c1
--- /dev/null
+++ b/apps/t3/app/pom.xml
@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2017 Open Networking Foundation
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.onosproject</groupId>
+        <artifactId>onos-apps</artifactId>
+        <version>1.14.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>onos-apps-t3-app</artifactId>
+    <packaging>bundle</packaging>
+
+    <description>Trellis Troubleshooting Toolkit</description>
+
+    <properties>
+        <onos.app.title>Trellis Troubleshooting Toolkit</onos.app.title>
+        <onos.app.category>Utilities</onos.app.category>
+        <onos.app.url>http://onosproject.org</onos.app.url>
+        <onos.app.readme>Static analysis of flows and groups to determine paths of packets.</onos.app.readme>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-apps-mcast-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-apps-route-service-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-apps-route-service-api</artifactId>
+            <version>${project.version}</version>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-apps-segmentrouting-app</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.osgi</groupId>
+            <artifactId>org.osgi.compendium</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-core-serializers</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-api</artifactId>
+            <scope>test</scope>
+            <classifier>tests</classifier>
+        </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-core-primitives</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-cli</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.karaf.shell</groupId>
+            <artifactId>org.apache.karaf.shell.console</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.felix</groupId>
+            <artifactId>org.apache.felix.scr.annotations</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-drivers-default</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onlab-misc</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>
+
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/api/GroupsInDevice.java b/apps/t3/app/src/main/java/org/onosproject/t3/api/GroupsInDevice.java
new file mode 100644
index 0000000..74203af
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/api/GroupsInDevice.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.api;
+
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.group.Group;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Class to represent the groups in a device for a given output and packet.
+ */
+//FIXME consider name change.
+public class GroupsInDevice {
+
+    private ConnectPoint output;
+    private List<Group> groups;
+    private TrafficSelector selector;
+
+    /**
+     * Saves the given groups for the output connect point and the selector.
+     *
+     * @param output   the output connect point
+     * @param groups   the groups
+     * @param selector the selector representing the final packet
+     */
+    public GroupsInDevice(ConnectPoint output, List<Group> groups, TrafficSelector selector) {
+
+        this.output = output;
+        this.groups = groups;
+        this.selector = selector;
+    }
+
+    /**
+     * Returns the output connect point.
+     *
+     * @return the connect point
+     */
+    public ConnectPoint getOutput() {
+        return output;
+    }
+
+    /**
+     * Returns the groups.
+     *
+     * @return groups.
+     */
+    public List<Group> getGroups() {
+        return groups;
+    }
+
+    /**
+     * Returns the final packet after traversing the network.
+     *
+     * @return the selector with packet info
+     */
+    public TrafficSelector getFinalPacket() {
+        return selector;
+    }
+
+    /**
+     * Updates the final packet.
+     *
+     * @param updatedPacket the updated final packet
+     */
+    public void setFinalPacket(TrafficSelector updatedPacket) {
+        selector = updatedPacket;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        GroupsInDevice that = (GroupsInDevice) o;
+
+        return Objects.equals(output, that.output) &&
+                Objects.equals(groups, that.groups) &&
+                Objects.equals(selector, that.selector);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(output, groups, selector);
+    }
+
+    @Override
+    public String toString() {
+        return "GroupsInDevice{" +
+
+                "output=" + output +
+                ", groups=" + groups +
+                ", selector=" + selector +
+                '}';
+    }
+
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/api/StaticPacketTrace.java b/apps/t3/app/src/main/java/org/onosproject/t3/api/StaticPacketTrace.java
new file mode 100644
index 0000000..22db007
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/api/StaticPacketTrace.java
@@ -0,0 +1,229 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.api;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.tuple.Pair;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.flow.FlowEntry;
+import org.onosproject.net.flow.TrafficSelector;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Encapsulates the result of tracing a packet (traffic selector) through
+ * the current topology.
+ */
+public class StaticPacketTrace {
+
+    private final TrafficSelector inPacket;
+    private final ConnectPoint in;
+    List<List<ConnectPoint>> completePaths;
+    private Map<DeviceId, List<GroupsInDevice>> outputsForDevice;
+    private Map<DeviceId, List<FlowEntry>> flowsForDevice;
+    private StringBuilder resultMessage;
+    private Pair<Host, Host> hosts;
+    private List<Boolean> success = new ArrayList<>();
+
+    /**
+     * Builds the trace with a given packet and a connect point.
+     *
+     * @param packet the packet to trace
+     * @param in     the initial connect point
+     */
+    public StaticPacketTrace(TrafficSelector packet, ConnectPoint in) {
+        this.inPacket = packet;
+        this.in = in;
+        completePaths = new ArrayList<>();
+        outputsForDevice = new HashMap<>();
+        flowsForDevice = new HashMap<>();
+        resultMessage = new StringBuilder();
+        hosts = null;
+    }
+
+    /**
+     * Builds the trace with a given packet and a connect point.
+     *
+     * @param packet the packet to trace
+     * @param in     the initial connect point
+     * @param hosts  pair of source and destination hosts
+     */
+    public StaticPacketTrace(TrafficSelector packet, ConnectPoint in, Pair<Host, Host> hosts) {
+        this.inPacket = packet;
+        this.in = in;
+        completePaths = new ArrayList<>();
+        outputsForDevice = new HashMap<>();
+        flowsForDevice = new HashMap<>();
+        resultMessage = new StringBuilder();
+        this.hosts = hosts;
+    }
+
+    /**
+     * Return the initial packet.
+     *
+     * @return the initial packet in the form of a selector.
+     */
+    public TrafficSelector getInitialPacket() {
+        return inPacket;
+    }
+
+    /**
+     * Returns the first connect point the packet came in through.
+     *
+     * @return the connect point
+     */
+    public ConnectPoint getInitialConnectPoint() {
+        return in;
+    }
+
+    /**
+     * Add a result message for the Trace.
+     *
+     * @param resultMessage the message
+     */
+    public void addResultMessage(String resultMessage) {
+        if (this.resultMessage.length() != 0) {
+            this.resultMessage.append("\n");
+        }
+        this.resultMessage.append(resultMessage);
+    }
+
+    /**
+     * Return the result message.
+     *
+     * @return the message
+     */
+    public String resultMessage() {
+        return resultMessage.toString();
+    }
+
+    /**
+     * Adds the groups for a given device.
+     *
+     * @param deviceId   the device
+     * @param outputPath the groups in device objects
+     */
+    public void addGroupOutputPath(DeviceId deviceId, GroupsInDevice outputPath) {
+        if (!outputsForDevice.containsKey(deviceId)) {
+            outputsForDevice.put(deviceId, new ArrayList<>());
+        }
+        outputsForDevice.get(deviceId).add(outputPath);
+    }
+
+    /**
+     * Returns all the possible group-based outputs for a given device.
+     *
+     * @param deviceId the device
+     * @return the list of Groups for this device.
+     */
+    public List<GroupsInDevice> getGroupOuputs(DeviceId deviceId) {
+        return outputsForDevice.get(deviceId) == null ? null : ImmutableList.copyOf(outputsForDevice.get(deviceId));
+    }
+
+    /**
+     * Adds a complete possible path.
+     *
+     * @param completePath the path
+     */
+    public void addCompletePath(List<ConnectPoint> completePath) {
+        completePaths.add(completePath);
+    }
+
+    /**
+     * Return all the possible path the packet can take through the network.
+     *
+     * @return a list of paths
+     */
+    public List<List<ConnectPoint>> getCompletePaths() {
+        return completePaths;
+    }
+
+    /**
+     * Add the flows traversed by the packet in a given device.
+     *
+     * @param deviceId the device considered
+     * @param flows    the flows
+     */
+    public void addFlowsForDevice(DeviceId deviceId, List<FlowEntry> flows) {
+        flowsForDevice.put(deviceId, flows);
+    }
+
+    /**
+     * Returns the flows matched by this trace's packet for a given device.
+     *
+     * @param deviceId the device
+     * @return the flows matched
+     */
+    public List<FlowEntry> getFlowsForDevice(DeviceId deviceId) {
+        return flowsForDevice.getOrDefault(deviceId, ImmutableList.of());
+    }
+
+    /**
+     * Return, if present, the two hosts at the endpoints of this trace.
+     *
+     * @return pair of source and destination hosts
+     */
+    public Optional<Pair<Host, Host>> getEndpointHosts() {
+        return Optional.ofNullable(hosts);
+    }
+
+    /**
+     * Sets the two hosts at the endpoints of this trace.
+     *
+     * @param endpointHosts pair of source and destination hosts
+     */
+    public void addEndpointHosts(Pair<Host, Host> endpointHosts) {
+        hosts = endpointHosts;
+    }
+
+    /**
+     * Return if all the possible paths of this trace are successful.
+     *
+     * @return true if all paths are successful
+     */
+    public boolean isSuccess() {
+        return !success.contains(false);
+    }
+
+    /**
+     * Sets if a path from this trace is successful.
+     *
+     * @param success true if a path of trace is successful.
+     */
+    public void setSuccess(boolean success) {
+        this.success.add(success);
+    }
+
+
+    @Override
+    public String toString() {
+        return "StaticPacketTrace{" +
+                "inPacket=" + inPacket +
+                ", in=" + in +
+                ", completePaths=" + completePaths +
+                ", outputsForDevice=" + outputsForDevice +
+                ", flowsForDevice=" + flowsForDevice +
+                ", resultMessage=" + resultMessage +
+                '}';
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/api/TroubleshootService.java b/apps/t3/app/src/main/java/org/onosproject/t3/api/TroubleshootService.java
new file mode 100644
index 0000000..5b19cc2
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/api/TroubleshootService.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.api;
+
+import org.onlab.packet.EthType;
+import org.onlab.packet.VlanId;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.HostId;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.t3.impl.Generator;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * API for troubleshooting services, providing static analysis of installed
+ * flows and groups.
+ */
+public interface TroubleshootService {
+
+    /**
+     * Requests a static trace be performed between all hosts in the network, given a type of traffic.
+     *
+     * @param type the etherType of the traffic we want to trace.
+     * @return a trace result
+     */
+    List<StaticPacketTrace> pingAll(EthType.EtherType type);
+
+    /**
+     * Requests a static trace be performed between all hosts in the network, given a type of traffic.
+     *
+     * @param type the etherType of the traffic we want to trace.
+     * @return a trace result
+     */
+    Generator<Set<StaticPacketTrace>> pingAllGenerator(EthType.EtherType type);
+
+    /**
+     * Requests a static trace be performed for all mcast Routes in the network.
+     *
+     * @param vlanId the vlanId configured for multicast.
+     * @return a set of trace result yielded one by one.
+     */
+    Generator<Set<StaticPacketTrace>> traceMcast(VlanId vlanId);
+
+    /**
+     * Requests a static trace be performed between the two hosts in the network, given a type of traffic.
+     *
+     * @param sourceHost      source host
+     * @param destinationHost destination host
+     * @param type            the etherType of the traffic we want to trace.
+     * @return a trace result
+     */
+    Set<StaticPacketTrace> trace(HostId sourceHost, HostId destinationHost, EthType.EtherType type);
+
+    /**
+     * Requests a static trace be performed for the given traffic selector
+     * starting at the given connect point.
+     *
+     * @param packet description of packet
+     * @param in     point at which packet starts
+     * @return a trace result
+     */
+    StaticPacketTrace trace(TrafficSelector packet, ConnectPoint in);
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/api/package-info.java b/apps/t3/app/src/main/java/org/onosproject/t3/api/package-info.java
new file mode 100644
index 0000000..152d568
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/api/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Troubleshooting toolkit for trellis fabrics API.
+ */
+package org.onosproject.t3.api;
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/cli/T3CliUtils.java b/apps/t3/app/src/main/java/org/onosproject/t3/cli/T3CliUtils.java
new file mode 100644
index 0000000..26c814e
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/cli/T3CliUtils.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2015-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.cli;
+
+import org.apache.commons.lang.StringUtils;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.group.GroupBucket;
+import org.onosproject.t3.api.GroupsInDevice;
+import org.onosproject.t3.api.StaticPacketTrace;
+
+import java.util.List;
+
+/**
+ * Class containing utility methods for T3 cli.
+ */
+final class T3CliUtils {
+
+    private T3CliUtils() {
+        //banning construction
+    }
+
+    private static final String FLOW_SHORT_FORMAT = "    %s, bytes=%s, packets=%s, "
+            + "table=%s, priority=%s, selector=%s, treatment=%s";
+
+    private static final String GROUP_FORMAT =
+            "   id=0x%s, state=%s, type=%s, bytes=%s, packets=%s, appId=%s, referenceCount=%s";
+    private static final String GROUP_BUCKET_FORMAT =
+            "       id=0x%s, bucket=%s, bytes=%s, packets=%s, actions=%s";
+
+    /**
+     * Builds a string output for the given trace for a specific level of verbosity.
+     *
+     * @param trace      the trace
+     * @param verbosity1 middle verbosity level
+     * @param verbosity2 high verbosity level
+     * @return a string representing the trace.
+     */
+    static String printTrace(StaticPacketTrace trace, boolean verbosity1, boolean verbosity2) {
+        StringBuilder tracePrint = new StringBuilder();
+        //Print based on verbosity
+        if (verbosity1) {
+            tracePrint = printTrace(trace, false, tracePrint);
+        } else if (verbosity2) {
+            tracePrint = printTrace(trace, true, tracePrint);
+        } else {
+            tracePrint.append("Paths");
+            tracePrint.append("\n");
+            List<List<ConnectPoint>> paths = trace.getCompletePaths();
+            for (List<ConnectPoint> path : paths) {
+                tracePrint.append(path);
+                tracePrint.append("\n");
+            }
+        }
+        tracePrint.append("Result: \n" + trace.resultMessage());
+        return tracePrint.toString();
+    }
+
+    //prints the trace
+    private static StringBuilder printTrace(StaticPacketTrace trace, boolean verbose, StringBuilder tracePrint) {
+        List<List<ConnectPoint>> paths = trace.getCompletePaths();
+        for (List<ConnectPoint> path : paths) {
+            tracePrint.append("Path " + path);
+            tracePrint.append("\n");
+            ConnectPoint previous = null;
+            if (path.size() == 1) {
+                ConnectPoint connectPoint = path.get(0);
+                tracePrint.append("Device " + connectPoint.deviceId());
+                tracePrint.append("\n");
+                tracePrint.append("Input from " + connectPoint);
+                tracePrint.append("\n");
+                tracePrint = printFlows(trace, verbose, connectPoint, tracePrint);
+                tracePrint = printGroups(trace, verbose, connectPoint, tracePrint);
+                tracePrint.append("\n");
+            } else {
+                for (ConnectPoint connectPoint : path) {
+                    if (previous == null || !previous.deviceId().equals(connectPoint.deviceId())) {
+                        tracePrint.append("Device " + connectPoint.deviceId());
+                        tracePrint.append("\n");
+                        tracePrint.append("    Input from " + connectPoint);
+                        tracePrint.append("\n");
+                        tracePrint = printFlows(trace, verbose, connectPoint, tracePrint);
+                    } else {
+                        tracePrint = printGroups(trace, verbose, connectPoint, tracePrint);
+                        tracePrint.append("    Output through " + connectPoint);
+                        tracePrint.append("\n");
+                    }
+                    previous = connectPoint;
+                }
+            }
+            tracePrint.append(StringUtils.leftPad("\n", 100, '-'));
+        }
+        return tracePrint;
+    }
+
+
+    //Prints the flows for a given trace and a specified level of verbosity
+    private static StringBuilder printFlows(StaticPacketTrace trace, boolean verbose, ConnectPoint connectPoint,
+                                            StringBuilder tracePrint) {
+        tracePrint.append("    Flows ");
+        tracePrint.append(trace.getFlowsForDevice(connectPoint.deviceId()).size());
+        tracePrint.append("    \n");
+        trace.getFlowsForDevice(connectPoint.deviceId()).forEach(f -> {
+            if (verbose) {
+                tracePrint.append("    " + String.format(FLOW_SHORT_FORMAT, f.state(), f.bytes(), f.packets(),
+                        f.table(), f.priority(), f.selector().criteria(),
+                        printTreatment(f.treatment())));
+                tracePrint.append("\n");
+            } else {
+                tracePrint.append(String.format("       flowId=%s, table=%s, selector=%s", f.id(), f.table(),
+                        f.selector().criteria()));
+                tracePrint.append("\n");
+            }
+        });
+        return tracePrint;
+    }
+
+    //Prints the groups for a given trace and a specified level of verbosity
+    private static StringBuilder printGroups(StaticPacketTrace trace, boolean verbose, ConnectPoint connectPoint,
+                                             StringBuilder tracePrint) {
+        List<GroupsInDevice> groupsInDevice = trace.getGroupOuputs(connectPoint.deviceId());
+        if (groupsInDevice != null) {
+            tracePrint.append("    Groups");
+            tracePrint.append("\n");
+            groupsInDevice.forEach(output -> {
+                if (output.getOutput().equals(connectPoint)) {
+                    output.getGroups().forEach(group -> {
+                        if (verbose) {
+                            tracePrint.append("    " + String.format(GROUP_FORMAT, Integer.toHexString(group.id().id()),
+                                    group.state(), group.type(), group.bytes(), group.packets(),
+                                    group.appId().name(), group.referenceCount()));
+                            tracePrint.append("\n");
+                            int i = 0;
+                            for (GroupBucket bucket : group.buckets().buckets()) {
+                                tracePrint.append("    " + String.format(GROUP_BUCKET_FORMAT,
+                                        Integer.toHexString(group.id().id()),
+                                        ++i, bucket.bytes(), bucket.packets(),
+                                        bucket.treatment().allInstructions()));
+                                tracePrint.append("\n");
+                            }
+                        } else {
+                            tracePrint.append("       groupId=" + group.id());
+                            tracePrint.append("\n");
+                        }
+                    });
+                    tracePrint.append("    Outgoing Packet " + output.getFinalPacket());
+                    tracePrint.append("\n");
+                }
+            });
+        }
+        return tracePrint;
+    }
+
+    private static String printTreatment(TrafficTreatment treatment) {
+        final String delimiter = ", ";
+        StringBuilder builder = new StringBuilder("[");
+        if (!treatment.immediate().isEmpty()) {
+            builder.append("immediate=" + treatment.immediate() + delimiter);
+        }
+        if (!treatment.deferred().isEmpty()) {
+            builder.append("deferred=" + treatment.deferred() + delimiter);
+        }
+        if (treatment.clearedDeferred()) {
+            builder.append("clearDeferred" + delimiter);
+        }
+        if (treatment.tableTransition() != null) {
+            builder.append("transition=" + treatment.tableTransition() + delimiter);
+        }
+        if (treatment.metered() != null) {
+            builder.append("meter=" + treatment.metered() + delimiter);
+        }
+        if (treatment.writeMetadata() != null) {
+            builder.append("metadata=" + treatment.writeMetadata() + delimiter);
+        }
+        // Chop off last delimiter
+        builder.replace(builder.length() - delimiter.length(), builder.length(), "");
+        builder.append("]");
+        return builder.toString();
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootMcastCommand.java b/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootMcastCommand.java
new file mode 100644
index 0000000..92ed92f
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootMcastCommand.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.cli;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.karaf.shell.commands.Command;
+import org.apache.karaf.shell.commands.Option;
+import org.onlab.packet.EthType;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.VlanId;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.EthTypeCriterion;
+import org.onosproject.net.flow.criteria.IPCriterion;
+import org.onosproject.t3.api.StaticPacketTrace;
+import org.onosproject.t3.api.TroubleshootService;
+import org.onosproject.t3.impl.Generator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Starts a Static Packet Trace for all the multicast routes in the system and prints the result.
+ */
+@Command(scope = "onos", name = "t3-troubleshoot-mcast",
+        description = "Traces all the mcast routes present in the system")
+public class TroubleshootMcastCommand extends AbstractShellCommand {
+
+
+    @Option(name = "-v", aliases = "--verbose", description = "Outputs trace for each mcast route")
+    private boolean verbosity1 = false;
+
+    @Option(name = "-vv", aliases = "--veryverbose", description = "Outputs middle level details of every trace")
+    private boolean verbosity2 = false;
+
+    @Option(name = "-vvv", aliases = "--veryveryverbose", description = "Outputs complete details of every trace")
+    private boolean verbosity3 = false;
+
+    @Option(name = "-vid", aliases = "--vlanId", description = "Vlan of incoming packet", valueToShowInHelp = "None")
+    String vlan = "None";
+
+
+    @Override
+    protected void execute() {
+        TroubleshootService service = get(TroubleshootService.class);
+        print("Tracing all Multicast routes in the System");
+
+        //Create the generator for the list of traces.
+        VlanId vlanId = vlan == null || vlan.isEmpty() ? VlanId.NONE : VlanId.vlanId(vlan);
+        Generator<Set<StaticPacketTrace>> generator = service.traceMcast(vlanId);
+        int totalTraces = 0;
+        List<StaticPacketTrace> failedTraces = new ArrayList<>();
+        StaticPacketTrace previousTrace = null;
+        while (generator.iterator().hasNext()) {
+            totalTraces++;
+            //Print also Route if possible or packet
+            Set<StaticPacketTrace> traces = generator.iterator().next();
+            if (!verbosity1 && !verbosity2 && !verbosity3) {
+                for (StaticPacketTrace trace : traces) {
+                    previousTrace = printTrace(previousTrace, trace);
+                    if (!trace.isSuccess()) {
+                        print("Failure: %s", trace.resultMessage());
+                        failedTraces.add(trace);
+                    } else {
+                        print("Success");
+                    }
+                }
+            } else {
+                traces.forEach(trace -> {
+                    print("Tracing packet: %s", trace.getInitialPacket());
+                    print("%s", T3CliUtils.printTrace(trace, verbosity2, verbosity3));
+                    print("%s", StringUtils.rightPad("", 125, '-'));
+                });
+            }
+        }
+
+        if (!verbosity1 && !verbosity2 && !verbosity3) {
+            if (failedTraces.size() != 0) {
+                print("%s", StringUtils.rightPad("", 125, '-'));
+                print("Failed Traces: %s", failedTraces.size());
+            }
+            previousTrace = null;
+            for (StaticPacketTrace trace : failedTraces) {
+                previousTrace = printTrace(previousTrace, trace);
+                print("Failure: %s", trace.resultMessage());
+            }
+            print("%s", StringUtils.rightPad("", 125, '-'));
+            print("Summary");
+            print("Total Traces %s, errors %s", totalTraces, failedTraces.size());
+            print("%s", StringUtils.rightPad("", 125, '-'));
+        }
+
+    }
+
+    private StaticPacketTrace printTrace(StaticPacketTrace previousTrace, StaticPacketTrace trace) {
+        if (previousTrace == null || !previousTrace.equals(trace)) {
+            print("%s", StringUtils.rightPad("", 125, '-'));
+            previousTrace = trace;
+            ConnectPoint initialConnectPoint = trace.getInitialConnectPoint();
+            TrafficSelector initialPacket = trace.getInitialPacket();
+            boolean isIPv4 = ((EthTypeCriterion) initialPacket.getCriterion(Criterion.Type.ETH_TYPE))
+                    .ethType().equals(EthType.EtherType.IPV4.ethType()
+                    );
+            IpPrefix group = ((IPCriterion) (isIPv4 ? trace.getInitialPacket()
+                    .getCriterion(Criterion.Type.IPV4_DST) : trace.getInitialPacket()
+                    .getCriterion(Criterion.Type.IPV6_DST))).ip();
+            print("Source %s, group %s", initialConnectPoint, group);
+        }
+        StringBuilder destinations = new StringBuilder();
+        if (trace.getCompletePaths().size() > 1) {
+            destinations.append("Sinks: ");
+        } else {
+            destinations.append("Sink: ");
+        }
+        trace.getCompletePaths().forEach(path -> {
+            destinations.append(path.get(path.size() - 1) + " ");
+        });
+        print("%s", destinations.toString());
+        return previousTrace;
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootPingAllCommand.java b/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootPingAllCommand.java
new file mode 100644
index 0000000..f9d7376
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootPingAllCommand.java
@@ -0,0 +1,228 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.cli;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.karaf.shell.commands.Command;
+import org.apache.karaf.shell.commands.Option;
+import org.onlab.packet.IpAddress;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.Host;
+import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.IPCriterion;
+import org.onosproject.t3.api.StaticPacketTrace;
+import org.onosproject.t3.api.TroubleshootService;
+import org.onosproject.t3.impl.Generator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import static java.lang.Thread.sleep;
+import static org.onlab.packet.EthType.EtherType;
+
+/**
+ * Starts a Static Packet Trace for a given input and prints the result.
+ */
+@Command(scope = "onos", name = "t3-troubleshoot-pingall",
+        description = "Traces a ping between all hosts in the system of a given ETH type")
+public class TroubleshootPingAllCommand extends AbstractShellCommand {
+
+    private static final String FMT_SHORT =
+            "id=%s, mac=%s, locations=%s, vlan=%s, ip(s)=%s";
+
+    @Option(name = "-et", aliases = "--ethType", description = "ETH Type", valueToShowInHelp = "ipv4")
+    String ethType = "ipv4";
+
+    @Option(name = "-v", aliases = "--verbose", description = "Outputs trace for each host to host combination")
+    private boolean verbosity1 = false;
+
+    @Option(name = "-vv", aliases = "--veryverbose", description = "Outputs details of every trace")
+    private boolean verbosity2 = false;
+
+    @Option(name = "-d", aliases = "--delay", description = "delay between host to host trace display")
+    private long delay = 0;
+
+    @Override
+    protected void execute() {
+        TroubleshootService service = get(TroubleshootService.class);
+
+        EtherType type = EtherType.valueOf(ethType.toUpperCase());
+
+        print("Tracing between all %s hosts", ethType);
+
+        if (!type.equals(EtherType.IPV4) && !type.equals(EtherType.IPV6)) {
+            print("Command only support IPv4 or IPv6");
+        } else {
+            //Create the generator for the list of traces.
+            Generator<Set<StaticPacketTrace>> generator = service.pingAllGenerator(type);
+            Host previousHost = null;
+            int totalTraces = 0;
+            List<StaticPacketTrace> failedTraces = new ArrayList<>();
+            boolean ipv4 = type.equals(EtherType.IPV4);
+            while (generator.iterator().hasNext()) {
+                Set<StaticPacketTrace> traces = generator.iterator().next();
+                totalTraces++;
+                for (StaticPacketTrace trace : traces) {
+                    //no verbosity is mininet style output
+                    if (!verbosity1 && !verbosity2) {
+                        if (trace.getEndpointHosts().isPresent()) {
+                            Host src = trace.getEndpointHosts().get().getLeft();
+                            if (previousHost == null || !previousHost.equals(src)) {
+                                print("%s", StringUtils.rightPad("", 125, '-'));
+                                previousHost = printSrc(trace, ipv4, src);
+                            }
+                            String host = getDstString(trace, ipv4, src);
+                            if (!trace.isSuccess()) {
+                                host = host + " " + trace.resultMessage();
+                                failedTraces.add(trace);
+                            }
+                            print("%s", host);
+                        }
+                    } else {
+                        print("%s", StringUtils.leftPad("", 125, '-'));
+
+                        if (trace.getInitialPacket() != null) {
+                            if (verbosity1) {
+                                printResultOnly(trace, ipv4);
+                            } else if (verbosity2) {
+                                printVerbose(trace);
+                            }
+                        } else {
+                            if (trace.getEndpointHosts().isPresent()) {
+                                Host source = trace.getEndpointHosts().get().getLeft();
+                                Host destination = trace.getEndpointHosts().get().getRight();
+                                print("Source %s --> Destination %s", source.id(), destination.id());
+                            }
+                            print("Error in obtaining trace: %s", trace.resultMessage());
+                        }
+                    }
+                }
+                try {
+                    sleep(delay);
+                } catch (InterruptedException e) {
+                    log.debug("interrupted while sleep");
+                }
+            }
+            print("%s", StringUtils.rightPad("", 125, '-'));
+            print("Failed Traces: %s", failedTraces.size());
+            print("%s", StringUtils.rightPad("", 125, '-'));
+            failedTraces.forEach(t -> {
+                if (t.getEndpointHosts().isPresent()) {
+                    printSrc(t, ipv4, t.getEndpointHosts().get().getLeft());
+                    String dst = getDstString(t, ipv4, t.getEndpointHosts().get().getRight());
+                    dst = dst + " " + t.resultMessage();
+                    print("%s", dst);
+                    print("%s", StringUtils.rightPad("", 125, '-'));
+                }
+            });
+            print("Summary");
+            print("Total Traces %s, errors %s", totalTraces, failedTraces.size());
+        }
+    }
+
+    private String getDstString(StaticPacketTrace trace, boolean ipv4, Host src) {
+        String host;
+        IpAddress ipAddress = getIpAddress(trace, ipv4, src, false);
+        if (ipAddress == null) {
+            host = String.format("       %s %s", trace.getEndpointHosts().get().getRight().id(),
+                    trace.isSuccess());
+        } else {
+            host = String.format("       %s (%s) %s",
+                    trace.getEndpointHosts().get().getRight().id(), ipAddress,
+                    trace.isSuccess());
+        }
+        return host;
+    }
+
+    private Host printSrc(StaticPacketTrace trace, boolean ipv4, Host src) {
+        Host previousHost;
+        IpAddress ipAddress = getIpAddress(trace, ipv4, src, true);
+        if (ipAddress == null) {
+            print("%s", src.id() + " -->");
+        } else {
+            print("%s (%s) -->", src.id(), ipAddress);
+        }
+        previousHost = src;
+        return previousHost;
+    }
+
+    private IpAddress getIpAddress(StaticPacketTrace trace, boolean ipv4, Host host, boolean src) {
+        IpAddress ipAddress;
+        if (ipv4) {
+            Criterion.Type type = src ? Criterion.Type.IPV4_SRC : Criterion.Type.IPV4_DST;
+            if (trace.getInitialPacket().getCriterion(type) != null) {
+                ipAddress = ((IPCriterion) trace.getInitialPacket()
+                        .getCriterion(type)).ip().address();
+            } else {
+                ipAddress = host.ipAddresses().stream().filter(IpAddress::isIp4)
+                        .findAny().orElseGet(null);
+            }
+        } else {
+            Criterion.Type type = src ? Criterion.Type.IPV6_SRC : Criterion.Type.IPV6_DST;
+            if (trace.getInitialPacket().getCriterion(type) != null) {
+                ipAddress = ((IPCriterion) trace.getInitialPacket()
+                        .getCriterion(type)).ip().address();
+            } else {
+                ipAddress = host.ipAddresses().stream().filter(IpAddress::isIp6)
+                        .findAny().orElseGet(null);
+            }
+        }
+        return ipAddress;
+    }
+
+    private void printResultOnly(StaticPacketTrace trace, boolean ipv4) {
+        if (trace.getEndpointHosts().isPresent()) {
+            Host source = trace.getEndpointHosts().get().getLeft();
+            Host destination = trace.getEndpointHosts().get().getRight();
+            IpAddress srcIP;
+            IpAddress dstIP;
+            if (ipv4 && trace.getInitialPacket().getCriterion(Criterion.Type.IPV4_SRC) != null) {
+                srcIP = ((IPCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.IPV4_SRC)).ip().address();
+                dstIP = ((IPCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.IPV4_DST)).ip().address();
+                print("Source %s (%s) --> Destination %s (%s)", source.id(), srcIP, destination.id(), dstIP);
+            } else if (trace.getInitialPacket().getCriterion(Criterion.Type.IPV6_SRC) != null) {
+                srcIP = ((IPCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.IPV6_SRC)).ip().address();
+                dstIP = ((IPCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.IPV6_DST)).ip().address();
+                print("Source %s (%s) --> Destination %s (%s)", source.id(), srcIP, destination.id(), dstIP);
+            } else {
+                print("Source %s --> Destination %s", source.id(), destination.id());
+            }
+            print("%s", trace.resultMessage());
+        } else {
+            print("Can't gather host information from trace");
+            print("%s", trace.resultMessage());
+        }
+    }
+
+    private void printVerbose(StaticPacketTrace trace) {
+        if (trace.getEndpointHosts().isPresent()) {
+            Host source = trace.getEndpointHosts().get().getLeft();
+            print("Source host %s", printHost(source));
+            Host destination = trace.getEndpointHosts().get().getRight();
+            print("Destination host %s", printHost(destination));
+        }
+        print("%s", trace.getInitialPacket());
+        print("%s", T3CliUtils.printTrace(trace, false, false));
+    }
+
+    private String printHost(Host host) {
+        return String.format(FMT_SHORT, host.id(), host.mac(),
+                host.locations(),
+                host.vlan(), host.ipAddresses());
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootSimpleTraceCommand.java b/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootSimpleTraceCommand.java
new file mode 100644
index 0000000..75356dc
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootSimpleTraceCommand.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.cli;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.apache.karaf.shell.commands.Option;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.cli.net.HostIdCompleter;
+import org.onosproject.net.HostId;
+import org.onosproject.t3.api.StaticPacketTrace;
+import org.onosproject.t3.api.TroubleshootService;
+
+import java.util.Set;
+
+import static org.onlab.packet.EthType.EtherType;
+
+/**
+ * Starts a Static Packet Trace for a given input and prints the result.
+ */
+@Command(scope = "onos", name = "t3-troubleshoot-simple",
+        description = "Given two hosts troubleshoots flows and groups between them, in case of segment routing")
+public class TroubleshootSimpleTraceCommand extends AbstractShellCommand {
+
+    // OSGi workaround to introduce package dependency
+    HostIdCompleter completer;
+    @Argument(index = 0, name = "one", description = "One host ID",
+            required = true, multiValued = false)
+    String srcHost = null;
+
+    @Argument(index = 1, name = "two", description = "Another host ID",
+            required = true, multiValued = false)
+    String dstHost = null;
+
+    @Option(name = "-v", aliases = "--verbose", description = "Outputs complete path")
+    private boolean verbosity1 = false;
+
+    @Option(name = "-vv", aliases = "--veryverbose", description = "Outputs flows and groups for every device")
+    private boolean verbosity2 = false;
+
+    @Option(name = "-et", aliases = "--ethType", description = "ETH Type", valueToShowInHelp = "ipv4")
+    String ethType = "ipv4";
+
+    @Override
+    protected void execute() {
+        TroubleshootService service = get(TroubleshootService.class);
+
+        EtherType type = EtherType.valueOf(ethType.toUpperCase());
+
+        //Printing the traced hosts
+        print("Tracing between: %s and %s", srcHost, dstHost);
+
+        //Build the traces
+        Set<StaticPacketTrace> traces = service.trace(HostId.hostId(srcHost), HostId.hostId(dstHost), type);
+        traces.forEach(trace -> {
+            if (trace.getInitialPacket() != null) {
+                print("Tracing Packet: %s", trace.getInitialPacket());
+                print("%s", T3CliUtils.printTrace(trace, verbosity1, verbosity2));
+            } else {
+                print("Cannot obtain trace between %s and %s", srcHost, dstHost);
+                print("Reason: %s", trace.resultMessage());
+            }
+        });
+
+
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootTraceCommand.java b/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootTraceCommand.java
new file mode 100644
index 0000000..e8b759d
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/cli/TroubleshootTraceCommand.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.cli;
+
+import com.google.common.base.Preconditions;
+import org.apache.karaf.shell.commands.Command;
+import org.apache.karaf.shell.commands.Option;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.MplsLabel;
+import org.onlab.packet.TpPort;
+import org.onlab.packet.VlanId;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.t3.api.StaticPacketTrace;
+import org.onosproject.t3.api.TroubleshootService;
+
+import static org.onlab.packet.EthType.EtherType;
+
+/**
+ * Starts a Static Packet Trace for a given input and prints the result.
+ */
+@Command(scope = "onos", name = "t3-troubleshoot",
+        description = "troubleshoots flows and groups between source and destination")
+public class TroubleshootTraceCommand extends AbstractShellCommand {
+
+
+    private static final String FLOW_SHORT_FORMAT = "    %s, bytes=%s, packets=%s, "
+            + "table=%s, priority=%s, selector=%s, treatment=%s";
+
+    private static final String GROUP_FORMAT =
+            "   id=0x%s, state=%s, type=%s, bytes=%s, packets=%s, appId=%s, referenceCount=%s";
+    private static final String GROUP_BUCKET_FORMAT =
+            "       id=0x%s, bucket=%s, bytes=%s, packets=%s, actions=%s";
+
+    private static final String CONTROLLER = "CONTROLLER";
+
+    @Option(name = "-v", aliases = "--verbose", description = "Outputs complete path")
+    private boolean verbosity1 = false;
+
+    @Option(name = "-vv", aliases = "--veryverbose", description = "Outputs flows and groups for every device")
+    private boolean verbosity2 = false;
+
+    @Option(name = "-s", aliases = "--srcIp", description = "Source IP")
+    String srcIp = null;
+
+    @Option(name = "-sp", aliases = "--srcPort", description = "Source Port", required = true)
+    String srcPort = null;
+
+    @Option(name = "-sm", aliases = "--srcMac", description = "Source MAC")
+    String srcMac = null;
+
+    @Option(name = "-et", aliases = "--ethType", description = "ETH Type", valueToShowInHelp = "ipv4")
+    String ethType = "ipv4";
+
+    @Option(name = "-stp", aliases = "--srcTcpPort", description = "Source TCP Port")
+    String srcTcpPort = null;
+
+    @Option(name = "-d", aliases = "--dstIp", description = "Destination IP")
+    String dstIp = null;
+
+    @Option(name = "-dm", aliases = "--dstMac", description = "Destination MAC")
+    String dstMac = null;
+
+    @Option(name = "-dtp", aliases = "--dstTcpPort", description = "destination TCP Port")
+    String dstTcpPort = null;
+
+    @Option(name = "-vid", aliases = "--vlanId", description = "Vlan of incoming packet", valueToShowInHelp = "None")
+    String vlan = "None";
+
+    @Option(name = "-ml", aliases = "--mplsLabel", description = "Mpls label of incoming packet")
+    String mplsLabel = null;
+
+    @Option(name = "-mb", aliases = "--mplsBos", description = "MPLS BOS")
+    String mplsBos = null;
+
+    @Option(name = "-ipp", aliases = "--ipProto", description = "IP Proto")
+    String ipProto = null;
+
+    @Option(name = "-udps", aliases = "--udpSrc", description = "UDP Source")
+    String udpSrc = null;
+
+    @Option(name = "-udpd", aliases = "--udpDst", description = "UDP Destination")
+    String udpDst = null;
+
+    @Override
+    protected void execute() {
+        TroubleshootService service = get(TroubleshootService.class);
+        String[] cpInfo = srcPort.split("/");
+        Preconditions.checkArgument(cpInfo.length == 2, "wrong format of source port");
+        ConnectPoint cp;
+        //Uses input port as a convenience to carry the Controller port, proper flood behaviour is handled in the
+        // troubleshoot manager.
+        if (cpInfo[1].equalsIgnoreCase(CONTROLLER)) {
+            cp = new ConnectPoint(DeviceId.deviceId(cpInfo[0]), PortNumber.CONTROLLER);
+        } else {
+            cp = ConnectPoint.deviceConnectPoint(srcPort);
+        }
+
+        EtherType type = EtherType.valueOf(ethType.toUpperCase());
+
+        //Input Port must be specified
+        TrafficSelector.Builder selectorBuilder = DefaultTrafficSelector.builder()
+                .matchInPort(cp.port());
+
+        if (srcIp != null) {
+            if (type.equals(EtherType.IPV6)) {
+                selectorBuilder.matchIPv6Src(IpAddress.valueOf(srcIp).toIpPrefix());
+            } else {
+                selectorBuilder.matchIPSrc(IpAddress.valueOf(srcIp).toIpPrefix());
+            }
+        }
+
+        if (srcMac != null) {
+            selectorBuilder.matchEthSrc(MacAddress.valueOf(srcMac));
+        }
+
+        //if EthType option is not specified using IPv4
+        selectorBuilder.matchEthType(type.ethType().toShort());
+
+        if (srcTcpPort != null) {
+            selectorBuilder.matchTcpSrc(TpPort.tpPort(Integer.parseInt(srcTcpPort)));
+        }
+
+        if (dstIp != null) {
+            if (type.equals(EtherType.IPV6)) {
+                selectorBuilder.matchIPv6Dst(IpAddress.valueOf(dstIp).toIpPrefix());
+            } else {
+                selectorBuilder.matchIPDst(IpAddress.valueOf(dstIp).toIpPrefix());
+            }
+        }
+
+        if (dstMac != null) {
+            selectorBuilder.matchEthDst(MacAddress.valueOf(dstMac));
+        }
+        if (dstTcpPort != null) {
+            selectorBuilder.matchTcpDst(TpPort.tpPort(Integer.parseInt(dstTcpPort)));
+        }
+
+        //if vlan option is not specified using NONE
+        selectorBuilder.matchVlanId(VlanId.vlanId(vlan));
+
+        if (mplsLabel != null) {
+            selectorBuilder.matchMplsLabel(MplsLabel.mplsLabel(Integer.parseInt(mplsLabel)));
+        }
+
+        if (mplsBos != null) {
+            selectorBuilder.matchMplsBos(Boolean.valueOf(mplsBos));
+        }
+
+        if (ipProto != null) {
+            selectorBuilder.matchIPProtocol(Byte.valueOf(ipProto));
+        }
+
+        if (udpSrc != null) {
+            selectorBuilder.matchUdpSrc(TpPort.tpPort(Integer.parseInt(udpSrc)));
+        }
+
+        if (udpDst != null) {
+            selectorBuilder.matchUdpDst(TpPort.tpPort(Integer.parseInt(udpDst)));
+        }
+
+
+        TrafficSelector packet = selectorBuilder.build();
+
+        //Printing the created packet
+        print("Tracing packet: %s", packet.criteria());
+
+        //Build the trace
+        StaticPacketTrace trace = service.trace(packet, cp);
+
+        print("%s", T3CliUtils.printTrace(trace, verbosity1, verbosity2));
+
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/cli/package-info.java b/apps/t3/app/src/main/java/org/onosproject/t3/cli/package-info.java
new file mode 100644
index 0000000..b956df2
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/cli/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Troubleshooting toolkit for trellis fabrics CLI.
+ */
+package org.onosproject.t3.cli;
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/impl/Generator.java b/apps/t3/app/src/main/java/org/onosproject/t3/impl/Generator.java
new file mode 100644
index 0000000..82ad628
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/impl/Generator.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.impl;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Generator class that yields instances of T type objects as soon as they are ready.
+ *
+ * @param <T> type of the object.
+ */
+public abstract class Generator<T> implements Iterable<T> {
+
+    private class Condition {
+        private boolean isSet;
+
+        synchronized void set() {
+            isSet = true;
+            notifyAll();
+        }
+
+        synchronized void await() throws InterruptedException {
+            try {
+
+                if (isSet) {
+                    return;
+                }
+
+                while (!isSet) {
+                    wait();
+                }
+            } finally {
+                isSet = false;
+            }
+        }
+    }
+
+    private static ThreadGroup threadGroup;
+
+    private Thread producer;
+    private boolean hasFinished;
+    private final Condition itemAvailableOrHasFinished = new Condition();
+    private final Condition itemRequested = new Condition();
+    private T nextItem;
+    private boolean nextItemAvailable;
+    private RuntimeException exceptionRaisedByProducer;
+
+    @Override
+    public Iterator<T> iterator() {
+        return new Iterator<T>() {
+            @Override
+            public boolean hasNext() {
+                return waitForNext();
+            }
+
+            @Override
+            public T next() {
+                if (!waitForNext()) {
+                    throw new NoSuchElementException();
+                }
+                nextItemAvailable = false;
+                return nextItem;
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+
+            private boolean waitForNext() {
+                if (nextItemAvailable) {
+                    return true;
+                }
+                if (hasFinished) {
+                    return false;
+                }
+                if (producer == null) {
+                    startProducer();
+                }
+                itemRequested.set();
+                try {
+                    itemAvailableOrHasFinished.await();
+                } catch (InterruptedException e) {
+                    hasFinished = true;
+                    producer.interrupt();
+                    try {
+                        producer.join();
+                    } catch (InterruptedException e1) {
+                        // Interrupting the broken thread
+                        Thread.currentThread().interrupt();
+                        throw new IllegalStateException(e1);
+                    }
+                }
+                if (exceptionRaisedByProducer != null) {
+                    throw exceptionRaisedByProducer;
+                }
+                return !hasFinished;
+            }
+        };
+    }
+
+    protected abstract void run() throws InterruptedException;
+
+    void yield(T element) throws InterruptedException {
+        nextItem = element;
+        nextItemAvailable = true;
+        itemAvailableOrHasFinished.set();
+        itemRequested.await();
+    }
+
+    private void startProducer() {
+        assert producer == null;
+        synchronized (this) {
+            if (threadGroup == null) {
+                threadGroup = new ThreadGroup("onos-t3-generator");
+            }
+        }
+        producer = new Thread(threadGroup, () -> {
+            try {
+                itemRequested.await();
+                Generator.this.run();
+            } catch (InterruptedException e) {
+                // Remaining steps in run() will shut down thread.
+            } catch (RuntimeException e) {
+                exceptionRaisedByProducer = e;
+            }
+            hasFinished = true;
+            itemAvailableOrHasFinished.set();
+        });
+        producer.setDaemon(true);
+        producer.start();
+    }
+}
\ No newline at end of file
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/impl/McastGenerator.java b/apps/t3/app/src/main/java/org/onosproject/t3/impl/McastGenerator.java
new file mode 100644
index 0000000..a5022a2
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/impl/McastGenerator.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.impl;
+
+import com.google.common.collect.ImmutableSet;
+import org.onlab.packet.EthType;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.VlanId;
+import org.onosproject.mcast.api.McastRouteData;
+import org.onosproject.mcast.api.MulticastRouteService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.t3.api.StaticPacketTrace;
+import org.slf4j.Logger;
+
+import java.util.Set;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Implementation of the generator class that yields a set of Packet Traces.
+ */
+public class McastGenerator extends Generator<Set<StaticPacketTrace>> {
+
+    private static final Logger log = getLogger(McastGenerator.class);
+    protected static final MacAddress IPV4_ADDRESS = MacAddress.valueOf("01:00:5E:00:00:00");
+    protected static final MacAddress IPV6_ADDRESS = MacAddress.valueOf("33:33:00:00:00:00");
+
+    private final MulticastRouteService mcastService;
+    private final TroubleshootManager manager;
+    private final VlanId vlanId;
+
+    /**
+     * Creates a generator for obtaining traces of all configured multicast routes.
+     *
+     * @param service the host service
+     * @param manager the troubleshoot manager issuing the request.
+     * @param vlanId  the multicast configured VlanId.
+     */
+    McastGenerator(MulticastRouteService service, TroubleshootManager manager, VlanId vlanId) {
+        this.mcastService = service;
+        this.manager = manager;
+        this.vlanId = vlanId;
+    }
+
+    @Override
+    protected void run() {
+        mcastService.getRoutes().forEach(route -> {
+            McastRouteData routeData = mcastService.routeData(route);
+            IpAddress group = route.group();
+            routeData.sources().forEach((host, sources) -> {
+                sources.forEach(source -> {
+                    TrafficSelector.Builder selector = DefaultTrafficSelector.builder()
+                            .matchVlanId(vlanId)
+                            .matchInPort(source.port());
+                    if (group.isIp4()) {
+                        selector.matchEthDst(IPV4_ADDRESS)
+                                .matchIPDst(group.toIpPrefix())
+                                .matchEthType(EthType.EtherType.IPV4.ethType().toShort());
+                    } else {
+                        selector.matchEthDst(IPV6_ADDRESS)
+                                .matchIPv6Dst(group.toIpPrefix())
+                                .matchEthType(EthType.EtherType.IPV6.ethType().toShort());
+                    }
+                    try {
+                        yield(ImmutableSet.of(manager.trace(selector.build(), source)));
+                    } catch (InterruptedException e) {
+                        log.warn("Interrupted generator", e.getMessage());
+                        log.debug("exception", e);
+                    }
+                });
+            });
+
+        });
+
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/impl/PingAllGenerator.java b/apps/t3/app/src/main/java/org/onosproject/t3/impl/PingAllGenerator.java
new file mode 100644
index 0000000..bf4d4b4
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/impl/PingAllGenerator.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.impl;
+
+import com.google.common.collect.Sets;
+import org.onlab.packet.EthType;
+import org.onlab.packet.IpAddress;
+import org.onosproject.net.host.HostService;
+import org.onosproject.t3.api.StaticPacketTrace;
+import org.slf4j.Logger;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Implementation of the generator class that yields a set of Packet Traces.
+ */
+public class PingAllGenerator extends Generator<Set<StaticPacketTrace>> {
+
+    private static final Logger log = getLogger(PingAllGenerator.class);
+
+    private final EthType.EtherType etherType;
+    private final HostService hostService;
+    private final TroubleshootManager manager;
+
+    /**
+     * Creates a generator for obtaining traces of pings between all the hosts in the network.
+     *
+     * @param etherType the type of traffic we are tracing.
+     * @param service   the host service
+     * @param manager   the troubleshoot manager issuing the request.
+     */
+    PingAllGenerator(EthType.EtherType etherType, HostService service, TroubleshootManager manager) {
+        this.etherType = etherType;
+        this.hostService = service;
+        this.manager = manager;
+    }
+
+    @Override
+    protected void run() throws InterruptedException {
+        hostService.getHosts().forEach(host -> {
+            List<IpAddress> ipAddresses = manager.getIpAddresses(host, etherType, false);
+            if (ipAddresses.size() > 0) {
+                //check if the host has only local IPs of that ETH type
+                boolean onlyLocalSrc = ipAddresses.size() == 1 && ipAddresses.get(0).isLinkLocal();
+                hostService.getHosts().forEach(hostToPing -> {
+                    List<IpAddress> ipAddressesToPing = manager.getIpAddresses(hostToPing, etherType, false);
+                    //check if the other host has only local IPs of that ETH type
+                    boolean onlyLocalDst = ipAddressesToPing.size() == 1 && ipAddressesToPing.get(0).isLinkLocal();
+                    boolean sameLocation = Sets.intersection(host.locations(), hostToPing.locations()).size() > 0;
+                    //Trace is done only if they are both local and under the same location
+                    // or not local and if they are not the same host.
+                    if (((sameLocation && onlyLocalDst && onlyLocalSrc) ||
+                            (!onlyLocalSrc && !onlyLocalDst && ipAddressesToPing.size() > 0))
+                            && !host.equals(hostToPing)) {
+                        try {
+                            yield(manager.trace(host.id(), hostToPing.id(), etherType));
+                        } catch (InterruptedException e) {
+                            log.warn("Interrupted generator", e.getMessage());
+                            log.debug("exception", e);
+                        }
+                    }
+                });
+            }
+        });
+
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/impl/Subnet.java b/apps/t3/app/src/main/java/org/onosproject/t3/impl/Subnet.java
new file mode 100644
index 0000000..ac5a741
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/impl/Subnet.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.impl;
+
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * Utility class to test if an Ip is in a given subnet.
+ */
+public class Subnet {
+    private final int bytesSubnetCount;
+    private final BigInteger bigMask;
+    private final BigInteger bigSubnetMasked;
+
+    /**
+     * Constructor for use via format "192.168.0.0/24" or "2001:db8:85a3:880:0:0:0:0/57".
+     * @param subnetAddress the address
+     * @param bits the mask
+     */
+    public Subnet(InetAddress subnetAddress, int bits) {
+        bytesSubnetCount = subnetAddress.getAddress().length; // 4 or 16
+        bigMask = BigInteger.valueOf(-1).shiftLeft(bytesSubnetCount * 8 - bits); // mask = -1 << 32 - bits
+        bigSubnetMasked = new BigInteger(subnetAddress.getAddress()).and(bigMask);
+    }
+
+    /**
+     * Constructor for use via format "192.168.0.0/255.255.255.0" or single address.
+     * @param subnetAddress the address
+     * @param mask the mask
+     */
+    public Subnet(InetAddress subnetAddress, InetAddress mask) {
+        bytesSubnetCount = subnetAddress.getAddress().length;
+        // no mask given case is handled here.
+        bigMask = null == mask ? BigInteger.valueOf(-1) : new BigInteger(mask.getAddress());
+        bigSubnetMasked = new BigInteger(subnetAddress.getAddress()).and(bigMask);
+    }
+
+    /**
+     * Subnet factory method.
+     *
+     * @param subnetMask format: "192.168.0.0/24" or "192.168.0.0/255.255.255.0"
+     *                   or single address or "2001:db8:85a3:880:0:0:0:0/57"
+     * @return a new instance
+     * @throws UnknownHostException thrown if unsupported subnet mask.
+     */
+    public static Subnet createInstance(String subnetMask)
+            throws UnknownHostException {
+        final String[] stringArr = subnetMask.split("/");
+        if (2 > stringArr.length) {
+            return new Subnet(InetAddress.getByName(stringArr[0]), (InetAddress) null);
+        } else if (stringArr[1].contains(".") || stringArr[1].contains(":")) {
+            return new Subnet(InetAddress.getByName(stringArr[0]), InetAddress.getByName(stringArr[1]));
+        } else {
+            return new Subnet(InetAddress.getByName(stringArr[0]), Integer.parseInt(stringArr[1]));
+        }
+    }
+
+    /**
+     * Tests if the address is in the given subnet.
+     * @param address the address to test.
+     * @return true if inside the subnet
+     */
+    public boolean isInSubnet(InetAddress address) {
+        byte[] bytesAddress = address.getAddress();
+        if (this.bytesSubnetCount != bytesAddress.length) {
+            return false;
+        }
+        BigInteger bigAddress = new BigInteger(bytesAddress);
+        return bigAddress.and(this.bigMask).equals(this.bigSubnetMasked);
+    }
+
+    @Override
+    public final boolean equals(Object obj) {
+        if (!(obj instanceof Subnet)) {
+            return false;
+        }
+        final Subnet other = (Subnet) obj;
+        return bigSubnetMasked.equals(other.bigSubnetMasked) &&
+                bigMask.equals(other.bigMask) &&
+                bytesSubnetCount == other.bytesSubnetCount;
+    }
+
+    @Override
+    public final int hashCode() {
+        return bytesSubnetCount;
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder buf = new StringBuilder();
+        bigInteger2IpString(buf, bigSubnetMasked, bytesSubnetCount);
+        buf.append('/');
+        bigInteger2IpString(buf, bigMask, bytesSubnetCount);
+        return buf.toString();
+    }
+
+    private void bigInteger2IpString(StringBuilder buf, BigInteger bigInteger, int displayBytes) {
+        boolean isIPv4 = 4 == displayBytes;
+        byte[] bytes = bigInteger.toByteArray();
+        int diffLen = displayBytes - bytes.length;
+        byte fillByte = 0 > (int) bytes[0] ? (byte) 0xFF : (byte) 0x00;
+
+        int integer;
+        for (int i = 0; i < displayBytes; i++) {
+            if (0 < i && !isIPv4 && i % 2 == 0) {
+                buf.append(':');
+            } else if (0 < i && isIPv4) {
+                buf.append('.');
+            }
+            integer = 0xFF & (i < diffLen ? fillByte : bytes[i - diffLen]);
+            if (!isIPv4 && 0x10 > integer) {
+                buf.append('0');
+            }
+            buf.append(isIPv4 ? integer : Integer.toHexString(integer));
+        }
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/impl/TroubleshootManager.java b/apps/t3/app/src/main/java/org/onosproject/t3/impl/TroubleshootManager.java
new file mode 100644
index 0000000..6db93bc
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/impl/TroubleshootManager.java
@@ -0,0 +1,1385 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.VlanId;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.mcast.api.MulticastRouteService;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.HostId;
+import org.onosproject.net.Link;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.config.ConfigException;
+import org.onosproject.net.config.NetworkConfigService;
+import org.onosproject.net.config.basics.InterfaceConfig;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.edge.EdgePortService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.FlowEntry;
+import org.onosproject.net.flow.FlowRule;
+import org.onosproject.net.flow.FlowRuleService;
+import org.onosproject.net.flow.IndexTableId;
+import org.onosproject.net.flow.TableId;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.criteria.Criteria;
+import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.EthCriterion;
+import org.onosproject.net.flow.criteria.EthTypeCriterion;
+import org.onosproject.net.flow.criteria.IPCriterion;
+import org.onosproject.net.flow.criteria.VlanIdCriterion;
+import org.onosproject.net.flow.instructions.Instruction;
+import org.onosproject.net.flow.instructions.Instructions;
+import org.onosproject.net.flow.instructions.Instructions.OutputInstruction;
+import org.onosproject.net.flow.instructions.L2ModificationInstruction;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupBucket;
+import org.onosproject.net.group.GroupService;
+import org.onosproject.net.host.HostService;
+import org.onosproject.net.host.InterfaceIpAddress;
+import org.onosproject.net.intf.Interface;
+import org.onosproject.net.link.LinkService;
+import org.onosproject.routeservice.ResolvedRoute;
+import org.onosproject.routeservice.RouteService;
+import org.onosproject.segmentrouting.config.SegmentRoutingDeviceConfig;
+import org.onosproject.t3.api.GroupsInDevice;
+import org.onosproject.t3.api.StaticPacketTrace;
+import org.onosproject.t3.api.TroubleshootService;
+import org.slf4j.Logger;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.onlab.packet.EthType.EtherType;
+import static org.onosproject.net.flow.TrafficSelector.Builder;
+import static org.onosproject.net.flow.instructions.Instructions.GroupInstruction;
+import static org.onosproject.net.flow.instructions.L2ModificationInstruction.ModEtherInstruction;
+import static org.onosproject.net.flow.instructions.L2ModificationInstruction.ModMplsHeaderInstruction;
+import static org.onosproject.net.flow.instructions.L2ModificationInstruction.ModMplsLabelInstruction;
+import static org.onosproject.net.flow.instructions.L2ModificationInstruction.ModVlanIdInstruction;
+import static org.onosproject.t3.impl.TroubleshootUtils.compareMac;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Manager to troubleshoot packets inside the network.
+ * Given a representation of a packet follows it's path in the network according to the existing flows and groups in
+ * the devices.
+ */
+@Service
+@Component(immediate = true)
+public class TroubleshootManager implements TroubleshootService {
+
+    private static final Logger log = getLogger(TroubleshootManager.class);
+
+    static final String PACKET_TO_CONTROLLER = "Packet goes to the controller";
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected FlowRuleService flowRuleService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected GroupService groupService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected LinkService linkService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DriverService driverService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected MastershipService mastershipService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected NetworkConfigService networkConfigService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected EdgePortService edgePortService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected RouteService routeService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected MulticastRouteService mcastService;
+
+    @Override
+    public List<StaticPacketTrace> pingAll(EtherType type) {
+        ImmutableList.Builder<StaticPacketTrace> tracesBuilder = ImmutableList.builder();
+        hostService.getHosts().forEach(host -> {
+            List<IpAddress> ipAddresses = getIpAddresses(host, type, false);
+            if (ipAddresses.size() > 0) {
+                //check if the host has only local IPs of that ETH type
+                boolean onlyLocalSrc = ipAddresses.size() == 1 && ipAddresses.get(0).isLinkLocal();
+                hostService.getHosts().forEach(hostToPing -> {
+                    List<IpAddress> ipAddressesToPing = getIpAddresses(hostToPing, type, false);
+                    //check if the other host has only local IPs of that ETH type
+                    boolean onlyLocalDst = ipAddressesToPing.size() == 1 && ipAddressesToPing.get(0).isLinkLocal();
+                    boolean sameLocation = Sets.intersection(host.locations(), hostToPing.locations()).size() > 0;
+                    //Trace is done only if they are both local and under the same location
+                    // or not local and if they are not the same host.
+                    if (((sameLocation && onlyLocalDst && onlyLocalSrc) ||
+                            (!onlyLocalSrc && !onlyLocalDst && ipAddressesToPing.size() > 0))
+                            && !host.equals(hostToPing)) {
+                        tracesBuilder.addAll(trace(host.id(), hostToPing.id(), type));
+                    }
+                });
+            }
+        });
+        return tracesBuilder.build();
+    }
+
+    @Override
+    public Generator<Set<StaticPacketTrace>> pingAllGenerator(EtherType type) {
+        return new PingAllGenerator(type, hostService, this);
+    }
+
+    @Override
+    public Generator<Set<StaticPacketTrace>> traceMcast(VlanId vlanId) {
+        return new McastGenerator(mcastService, this, vlanId);
+    }
+
+    @Override
+    public Set<StaticPacketTrace> trace(HostId sourceHost, HostId destinationHost, EtherType etherType) {
+        Host source = hostService.getHost(sourceHost);
+        Host destination = hostService.getHost(destinationHost);
+
+        //Temporary trace to fail in case we don't have enough information or what is provided is incoherent
+        StaticPacketTrace failTrace = new StaticPacketTrace(null, null, Pair.of(source, destination));
+
+        if (source == null) {
+            failTrace.addResultMessage("Source Host " + sourceHost + " does not exist");
+            failTrace.setSuccess(false);
+
+            return ImmutableSet.of(failTrace);
+        }
+
+        if (destination == null) {
+            failTrace.addResultMessage("Destination Host " + destinationHost + " does not exist");
+            failTrace.setSuccess(false);
+            return ImmutableSet.of(failTrace);
+        }
+
+        TrafficSelector.Builder selectorBuilder = DefaultTrafficSelector.builder()
+                .matchEthType(etherType.ethType().toShort())
+                .matchEthDst(source.mac())
+                .matchVlanId(source.vlan());
+
+
+        try {
+            ImmutableSet.Builder<StaticPacketTrace> traces = ImmutableSet.builder();
+            //if the location deviceId is the same, the two hosts are under same subnet and vlan on the interface
+            // we are under same leaf so it's L2 Unicast.
+            if (areBridged(source, destination)) {
+                selectorBuilder.matchEthDst(destination.mac());
+                source.locations().forEach(hostLocation -> {
+                    selectorBuilder.matchInPort(hostLocation.port());
+                    StaticPacketTrace trace = trace(selectorBuilder.build(), hostLocation);
+                    trace.addEndpointHosts(Pair.of(source, destination));
+                    traces.add(trace);
+                });
+                //The destination host is not dual homed, if it is the other path might be done through routing.
+                if (destination.locations().size() == 1) {
+                    return traces.build();
+                }
+            }
+            //handle the IPs for src and dst in case of L3
+            if (etherType.equals(EtherType.IPV4) || etherType.equals(EtherType.IPV6)) {
+
+                //Match on the source IP
+                if (!matchIP(source, failTrace, selectorBuilder, etherType, true)) {
+                    return ImmutableSet.of(failTrace);
+                }
+
+                //Match on destination IP
+                if (!matchIP(destination, failTrace, selectorBuilder, etherType, false)) {
+                    return ImmutableSet.of(failTrace);
+                }
+
+            } else {
+                failTrace.addResultMessage("Host based trace supports only IPv4 or IPv6 as EtherType, " +
+                        "please use packet based");
+                failTrace.setSuccess(false);
+                return ImmutableSet.of(failTrace);
+            }
+
+            //l3 unicast, we get the dst mac of the leaf the source is connected to from netcfg
+            SegmentRoutingDeviceConfig segmentRoutingConfig = networkConfigService.getConfig(source.location()
+                    .deviceId(), SegmentRoutingDeviceConfig.class);
+            if (segmentRoutingConfig != null) {
+                selectorBuilder.matchEthDst(segmentRoutingConfig.routerMac());
+            } else {
+                failTrace.addResultMessage("Can't get " + source.location().deviceId() +
+                        " router MAC from segment routing config can't perform L3 tracing.");
+                failTrace.setSuccess(false);
+            }
+            source.locations().forEach(hostLocation -> {
+                selectorBuilder.matchInPort(hostLocation.port());
+                StaticPacketTrace trace = trace(selectorBuilder.build(), hostLocation);
+                trace.addEndpointHosts(Pair.of(source, destination));
+                traces.add(trace);
+            });
+            return traces.build();
+
+        } catch (ConfigException e) {
+            failTrace.addResultMessage("Can't get config " + e.getMessage());
+            return ImmutableSet.of(failTrace);
+        }
+    }
+
+    /**
+     * Matches src and dst IPs based on host information.
+     *
+     * @param host            the host
+     * @param failTrace       the trace to use in case of failure
+     * @param selectorBuilder the packet we are building to trace
+     * @param etherType       the traffic type
+     * @param src             is this src host or dst host
+     * @return true if properly matched
+     */
+    private boolean matchIP(Host host, StaticPacketTrace failTrace, Builder selectorBuilder,
+                            EtherType etherType, boolean src) {
+        List<IpAddress> ips = getIpAddresses(host, etherType, true);
+
+        if (ips.size() > 0) {
+            if (etherType.equals(EtherType.IPV4)) {
+                if (src) {
+                    selectorBuilder.matchIPSrc(ips.get(0).toIpPrefix());
+                } else {
+                    selectorBuilder.matchIPDst(ips.get(0).toIpPrefix());
+                }
+            } else if (etherType.equals(EtherType.IPV6)) {
+                if (src) {
+                    selectorBuilder.matchIPv6Src(ips.get(0).toIpPrefix());
+                } else {
+                    selectorBuilder.matchIPv6Dst(ips.get(0).toIpPrefix());
+                }
+            }
+        } else {
+            failTrace.addResultMessage("Host " + host + " has no " + etherType + " address");
+            failTrace.setSuccess(false);
+            return false;
+        }
+        return true;
+    }
+
+    List<IpAddress> getIpAddresses(Host host, EtherType etherType, boolean checklocal) {
+        return host.ipAddresses().stream().filter(ipAddress -> {
+            boolean correctIp = false;
+            if (etherType.equals(EtherType.IPV4)) {
+                correctIp = ipAddress.isIp4();
+            } else if (etherType.equals(EtherType.IPV6)) {
+                correctIp = ipAddress.isIp6();
+            }
+            if (checklocal) {
+                correctIp = correctIp && !ipAddress.isLinkLocal();
+            }
+            return correctIp;
+        }).collect(Collectors.toList());
+    }
+
+    /**
+     * Checks that two hosts are bridged (L2Unicast).
+     *
+     * @param source      the source host
+     * @param destination the destination host
+     * @return true if bridged.
+     * @throws ConfigException if config can't be properly retrieved
+     */
+    private boolean areBridged(Host source, Host destination) throws ConfigException {
+
+        //If the locations is not the same we don't even check vlan or subnets
+        if (Collections.disjoint(source.locations(), destination.locations())) {
+            return false;
+        }
+
+        if (!source.vlan().equals(VlanId.NONE) && !destination.vlan().equals(VlanId.NONE)
+                && !source.vlan().equals(destination.vlan())) {
+            return false;
+        }
+
+        InterfaceConfig interfaceCfgH1 = networkConfigService.getConfig(source.location(), InterfaceConfig.class);
+        InterfaceConfig interfaceCfgH2 = networkConfigService.getConfig(destination.location(), InterfaceConfig.class);
+        if (interfaceCfgH1 != null && interfaceCfgH2 != null) {
+
+            //following can be optimized but for clarity is left as is
+            Interface intfH1 = interfaceCfgH1.getInterfaces().stream().findFirst().get();
+            Interface intfH2 = interfaceCfgH2.getInterfaces().stream().findFirst().get();
+
+            if (source.vlan().equals(VlanId.NONE) && !destination.vlan().equals(VlanId.NONE)) {
+                return intfH1.vlanUntagged().equals(destination.vlan()) ||
+                        intfH1.vlanNative().equals(destination.vlan());
+            }
+
+            if (!source.vlan().equals(VlanId.NONE) && destination.vlan().equals(VlanId.NONE)) {
+                return intfH2.vlanUntagged().equals(source.vlan()) ||
+                        intfH2.vlanNative().equals(source.vlan());
+            }
+
+            if (!intfH1.vlanNative().equals(intfH2.vlanNative())) {
+                return false;
+            }
+
+            if (!intfH1.vlanUntagged().equals(intfH2.vlanUntagged())) {
+                return false;
+            }
+
+            List<InterfaceIpAddress> intersection = new ArrayList<>(intfH1.ipAddressesList());
+            intersection.retainAll(intfH2.ipAddressesList());
+            if (intersection.size() == 0) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public StaticPacketTrace trace(TrafficSelector packet, ConnectPoint in) {
+        log.info("Tracing packet {} coming in through {}", packet, in);
+        //device must exist in ONOS
+        Preconditions.checkNotNull(deviceService.getDevice(in.deviceId()),
+                "Device " + in.deviceId() + " must exist in ONOS");
+
+        StaticPacketTrace trace = new StaticPacketTrace(packet, in);
+        boolean isDualHomed = getHosts(trace).stream().anyMatch(host -> host.locations().size() > 1);
+        //FIXME this can be done recursively
+        //Building output connect Points
+        List<ConnectPoint> path = new ArrayList<>();
+        trace = traceInDevice(trace, packet, in, isDualHomed, path);
+        trace = getTrace(path, in, trace, isDualHomed);
+        return trace;
+    }
+
+    /**
+     * Computes a trace for a give packet that start in the network at the given connect point.
+     *
+     * @param completePath the path traversed by the packet
+     * @param in           the input connect point
+     * @param trace        the trace to build
+     * @param isDualHomed  true if the trace we are doing starts or ends in a dual homed host
+     * @return the build trace for that packet.
+     */
+    private StaticPacketTrace getTrace(List<ConnectPoint> completePath, ConnectPoint in, StaticPacketTrace trace,
+                                       boolean isDualHomed) {
+
+        log.debug("------------------------------------------------------------");
+
+        //if the trace already contains the input connect point there is a loop
+        if (pathContainsDevice(completePath, in.deviceId())) {
+            trace.addResultMessage("Loop encountered in device " + in.deviceId());
+            completePath.add(in);
+            trace.addCompletePath(completePath);
+            trace.setSuccess(false);
+            return trace;
+        }
+
+        //let's add the input connect point
+        completePath.add(in);
+
+        //If the trace has no outputs for the given input we stop here
+        if (trace.getGroupOuputs(in.deviceId()) == null) {
+            computePath(completePath, trace, null);
+            trace.addResultMessage("No output out of device " + in.deviceId() + ". Packet is dropped");
+            trace.setSuccess(false);
+            return trace;
+        }
+
+        //If the trace has outputs we analyze them all
+        for (GroupsInDevice outputPath : trace.getGroupOuputs(in.deviceId())) {
+
+            ConnectPoint cp = outputPath.getOutput();
+            log.debug("Connect point in {}", in);
+            log.debug("Output path {}", cp);
+            log.debug("{}", outputPath.getFinalPacket());
+
+            //Hosts for the the given output
+            Set<Host> hostsList = hostService.getConnectedHosts(cp);
+            //Hosts queried from the original ip or mac
+            Set<Host> hosts = getHosts(trace);
+
+            if (in.equals(cp) && trace.getInitialPacket().getCriterion(Criterion.Type.VLAN_VID) != null &&
+                    outputPath.getFinalPacket().getCriterion(Criterion.Type.VLAN_VID) != null
+                    && ((VlanIdCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.VLAN_VID)).vlanId()
+                    .equals(((VlanIdCriterion) outputPath.getFinalPacket().getCriterion(Criterion.Type.VLAN_VID))
+                            .vlanId())) {
+                if (trace.getGroupOuputs(in.deviceId()).size() == 1 &&
+                        computePath(completePath, trace, outputPath.getOutput())) {
+                    trace.addResultMessage("Connect point out " + cp + " is same as initial input " + in);
+                    trace.setSuccess(false);
+                }
+            } else if (!Collections.disjoint(hostsList, hosts)) {
+                //If the two host collections contain the same item it means we reached the proper output
+                log.debug("Stopping here because host is expected destination {}, reached through", completePath);
+                if (computePath(completePath, trace, outputPath.getOutput())) {
+                    trace.addResultMessage("Reached required destination Host " + cp);
+                    trace.setSuccess(true);
+                }
+                break;
+            } else if (cp.port().equals(PortNumber.CONTROLLER)) {
+
+                //Getting the master when the packet gets sent as packet in
+                NodeId master = mastershipService.getMasterFor(cp.deviceId());
+                trace.addResultMessage(PACKET_TO_CONTROLLER + " " + master.id());
+                computePath(completePath, trace, outputPath.getOutput());
+                handleVlanToController(outputPath, trace);
+
+            } else if (linkService.getEgressLinks(cp).size() > 0) {
+
+                //TODO this can be optimized if we use a Tree structure for paths.
+                //if we already have outputs let's check if the one we are considering starts from one of the devices
+                // in any of the ones we have.
+                if (trace.getCompletePaths().size() > 0) {
+                    ConnectPoint inputForOutput = null;
+                    List<ConnectPoint> previousPath = new ArrayList<>();
+                    for (List<ConnectPoint> path : trace.getCompletePaths()) {
+                        for (ConnectPoint connect : path) {
+                            //if the path already contains the input for the output we've found we use it
+                            if (connect.equals(in)) {
+                                inputForOutput = connect;
+                                previousPath = path;
+                                break;
+                            }
+                        }
+                    }
+
+                    //we use the pre-existing path up to the point we fork to a new output
+                    if (inputForOutput != null && completePath.contains(inputForOutput)) {
+                        List<ConnectPoint> temp = new ArrayList<>(previousPath);
+                        temp = temp.subList(0, previousPath.indexOf(inputForOutput) + 1);
+                        if (completePath.containsAll(temp)) {
+                            completePath = temp;
+                        }
+                    }
+                }
+
+                //let's add the ouput for the input
+                completePath.add(cp);
+                //let's compute the links for the given output
+                Set<Link> links = linkService.getEgressLinks(cp);
+                log.debug("Egress Links {}", links);
+                //For each link we trace the corresponding device
+                for (Link link : links) {
+                    ConnectPoint dst = link.dst();
+                    //change in-port to the dst link in port
+                    Builder updatedPacket = DefaultTrafficSelector.builder();
+                    outputPath.getFinalPacket().criteria().forEach(updatedPacket::add);
+                    updatedPacket.add(Criteria.matchInPort(dst.port()));
+                    log.debug("DST Connect Point {}", dst);
+                    //build the elements for that device
+                    traceInDevice(trace, updatedPacket.build(), dst, isDualHomed, completePath);
+                    //continue the trace along the path
+                    getTrace(completePath, dst, trace, isDualHomed);
+                }
+            } else if (edgePortService.isEdgePoint(outputPath.getOutput()) &&
+                    trace.getInitialPacket().getCriterion(Criterion.Type.ETH_DST) != null &&
+                    ((EthCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.ETH_DST))
+                            .mac().isMulticast()) {
+                trace.addResultMessage("Packet is multicast and reached output " + outputPath.getOutput() +
+                        " which is enabled and is edge port");
+                trace.setSuccess(true);
+                computePath(completePath, trace, outputPath.getOutput());
+                if (!hasOtherOutput(in.deviceId(), trace, outputPath.getOutput())) {
+                    return trace;
+                }
+            } else if (deviceService.getPort(cp) != null && deviceService.getPort(cp).isEnabled()) {
+                EthTypeCriterion ethTypeCriterion = (EthTypeCriterion) trace.getInitialPacket()
+                        .getCriterion(Criterion.Type.ETH_TYPE);
+                //We treat as correct output only if it's not LLDP or BDDP
+                if (!(ethTypeCriterion.ethType().equals(EtherType.LLDP.ethType())
+                        && !ethTypeCriterion.ethType().equals(EtherType.BDDP.ethType()))) {
+                    if (computePath(completePath, trace, outputPath.getOutput())) {
+                        if (hostsList.isEmpty()) {
+                            trace.addResultMessage("Packet is " + ((EthTypeCriterion) outputPath.getFinalPacket()
+                                    .getCriterion(Criterion.Type.ETH_TYPE)).ethType() + " and reached " +
+                                    cp + " with no hosts connected ");
+                        } else {
+                            IpAddress ipAddress = null;
+                            if (trace.getInitialPacket().getCriterion(Criterion.Type.IPV4_DST) != null) {
+                                ipAddress = ((IPCriterion) trace.getInitialPacket()
+                                        .getCriterion(Criterion.Type.IPV4_DST)).ip().address();
+                            } else if (trace.getInitialPacket().getCriterion(Criterion.Type.IPV6_DST) != null) {
+                                ipAddress = ((IPCriterion) trace.getInitialPacket()
+                                        .getCriterion(Criterion.Type.IPV6_DST)).ip().address();
+                            }
+                            if (ipAddress != null) {
+                                IpAddress finalIpAddress = ipAddress;
+                                if (hostsList.stream().anyMatch(host -> host.ipAddresses().contains(finalIpAddress)) ||
+                                        hostService.getHostsByIp(finalIpAddress).isEmpty()) {
+                                    trace.addResultMessage("Packet is " +
+                                            ((EthTypeCriterion) outputPath.getFinalPacket()
+                                                    .getCriterion(Criterion.Type.ETH_TYPE)).ethType() +
+                                            " and reached " + cp + " with hosts " + hostsList);
+                                } else {
+                                    trace.addResultMessage("Wrong output " + cp + " for required destination ip " +
+                                            ipAddress);
+                                    trace.setSuccess(false);
+                                }
+                            } else {
+                                trace.addResultMessage("Packet is " + ((EthTypeCriterion) outputPath.getFinalPacket()
+                                        .getCriterion(Criterion.Type.ETH_TYPE)).ethType() + " and reached " +
+                                        cp + " with hosts " + hostsList);
+                            }
+                        }
+                        trace.setSuccess(true);
+                    }
+                }
+
+            } else {
+                computePath(completePath, trace, cp);
+                trace.setSuccess(false);
+                if (deviceService.getPort(cp) == null) {
+                    //Port is not existant on device.
+                    log.warn("Port {} is not available on device.", cp);
+                    trace.addResultMessage("Port " + cp + "is not available on device. Packet is dropped");
+                } else {
+                    //No links means that the packet gets dropped.
+                    log.warn("No links out of {}", cp);
+                    trace.addResultMessage("No links depart from " + cp + ". Packet is dropped");
+                }
+            }
+        }
+        return trace;
+    }
+
+
+    /**
+     * If the initial packet comes tagged with a Vlan we output it with that to ONOS.
+     * If ONOS applied a vlan we remove it.
+     *
+     * @param outputPath the output
+     * @param trace      the trace we are building
+     */
+
+    private void handleVlanToController(GroupsInDevice outputPath, StaticPacketTrace trace) {
+
+        VlanIdCriterion initialVid = (VlanIdCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.VLAN_VID);
+        VlanIdCriterion finalVid = (VlanIdCriterion) outputPath.getFinalPacket().getCriterion(Criterion.Type.VLAN_VID);
+
+        if (initialVid != null && !initialVid.equals(finalVid) && initialVid.vlanId().equals(VlanId.NONE)) {
+
+            Set<Criterion> finalCriteria = new HashSet<>(outputPath.getFinalPacket().criteria());
+            //removing the final vlanId
+            finalCriteria.remove(finalVid);
+            Builder packetUpdated = DefaultTrafficSelector.builder();
+            finalCriteria.forEach(packetUpdated::add);
+            //Initial was none so we set it to that
+            packetUpdated.add(Criteria.matchVlanId(VlanId.NONE));
+            //Update final packet
+            outputPath.setFinalPacket(packetUpdated.build());
+        }
+    }
+
+    /**
+     * Checks if the device has other outputs than the given connect point.
+     *
+     * @param inDeviceId the device
+     * @param trace      the trace we are building
+     * @param cp         an output connect point
+     * @return true if the device has other outputs.
+     */
+    private boolean hasOtherOutput(DeviceId inDeviceId, StaticPacketTrace trace, ConnectPoint cp) {
+        return trace.getGroupOuputs(inDeviceId).stream().filter(groupsInDevice -> {
+            return !groupsInDevice.getOutput().equals(cp);
+        }).count() > 0;
+    }
+
+    /**
+     * Checks if the path contains the device.
+     *
+     * @param completePath the path
+     * @param deviceId     the device to check
+     * @return true if the path contains the device
+     */
+    //TODO might prove costly, improvement: a class with both CPs and DeviceIds point.
+    private boolean pathContainsDevice(List<ConnectPoint> completePath, DeviceId deviceId) {
+        for (ConnectPoint cp : completePath) {
+            if (cp.deviceId().equals(deviceId)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Gets the hosts for the given initial packet.
+     *
+     * @param trace the trace we are building
+     * @return set of the hosts we are trying to reach
+     */
+    private Set<Host> getHosts(StaticPacketTrace trace) {
+        IPCriterion ipv4Criterion = ((IPCriterion) trace.getInitialPacket()
+                .getCriterion(Criterion.Type.IPV4_DST));
+        IPCriterion ipv6Criterion = ((IPCriterion) trace.getInitialPacket()
+                .getCriterion(Criterion.Type.IPV6_DST));
+        Set<Host> hosts = new HashSet<>();
+        if (ipv4Criterion != null) {
+            hosts.addAll(hostService.getHostsByIp(ipv4Criterion.ip().address()));
+        }
+        if (ipv6Criterion != null) {
+            hosts.addAll(hostService.getHostsByIp(ipv6Criterion.ip().address()));
+        }
+        EthCriterion ethCriterion = ((EthCriterion) trace.getInitialPacket()
+                .getCriterion(Criterion.Type.ETH_DST));
+        if (ethCriterion != null) {
+            hosts.addAll(hostService.getHostsByMac(ethCriterion.mac()));
+        }
+        return hosts;
+    }
+
+    /**
+     * Computes the list of traversed connect points.
+     *
+     * @param completePath the list of devices
+     * @param trace        the trace we are building
+     * @param output       the final output connect point
+     */
+    private boolean computePath(List<ConnectPoint> completePath, StaticPacketTrace trace, ConnectPoint output) {
+        List<ConnectPoint> traverseList = new ArrayList<>();
+        if (!completePath.contains(trace.getInitialConnectPoint())) {
+            traverseList.add(trace.getInitialConnectPoint());
+        }
+
+        if (output != null && trace.getInitialConnectPoint().deviceId().equals(output.deviceId())) {
+            trace.addCompletePath(ImmutableList.of(trace.getInitialConnectPoint(), output));
+            return true;
+        }
+
+        traverseList.addAll(completePath);
+        if (output != null && !completePath.contains(output)) {
+            traverseList.add(output);
+        }
+        if (!trace.getCompletePaths().contains(traverseList)) {
+            trace.addCompletePath(ImmutableList.copyOf(traverseList));
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Traces the packet inside a device starting from an input connect point.
+     *
+     * @param trace        the trace we are building
+     * @param packet       the packet we are tracing
+     * @param in           the input connect point.
+     * @param isDualHomed  true if the trace we are doing starts or ends in a dual homed host
+     * @param completePath the path up until this device
+     * @return updated trace
+     */
+    private StaticPacketTrace traceInDevice(StaticPacketTrace trace, TrafficSelector packet, ConnectPoint in,
+                                            boolean isDualHomed, List<ConnectPoint> completePath) {
+
+        boolean multipleRoutes = false;
+        if (trace.getGroupOuputs(in.deviceId()) != null) {
+            multipleRoutes = multipleRoutes(trace);
+        }
+        if (trace.getGroupOuputs(in.deviceId()) != null && !isDualHomed && !multipleRoutes) {
+            log.debug("Trace already contains device and given outputs");
+            return trace;
+        }
+
+        log.debug("Packet {} coming in from {}", packet, in);
+
+        //if device is not available exit here.
+        if (!deviceService.isAvailable(in.deviceId())) {
+            trace.addResultMessage("Device is offline " + in.deviceId());
+            computePath(completePath, trace, null);
+            return trace;
+        }
+
+        //handle when the input is the controller
+        //NOTE, we are using the input port as a convenience to carry the CONTROLLER port number even if
+        // a packet in from the controller will not actually traverse the pipeline and have no such notion
+        // as the input port.
+        if (in.port().equals(PortNumber.CONTROLLER)) {
+            StaticPacketTrace outputTrace = inputFromController(trace, in);
+            if (outputTrace != null) {
+                return trace;
+            }
+        }
+
+        List<FlowEntry> flows = new ArrayList<>();
+        List<FlowEntry> outputFlows = new ArrayList<>();
+        List<Instruction> deferredInstructions = new ArrayList<>();
+
+        FlowEntry nextTableIdEntry = findNextTableIdEntry(in.deviceId(), -1);
+        if (nextTableIdEntry == null) {
+            trace.addResultMessage("No flow rules for device " + in.deviceId() + ". Aborting");
+            computePath(completePath, trace, null);
+            trace.setSuccess(false);
+            return trace;
+        }
+        TableId tableId = nextTableIdEntry.table();
+        FlowEntry flowEntry;
+        boolean output = false;
+        while (!output) {
+            log.debug("Searching a Flow Entry on table {} for packet {}", tableId, packet);
+            //get the rule that matches the incoming packet
+            flowEntry = matchHighestPriority(packet, in, tableId);
+            log.debug("Found Flow Entry {}", flowEntry);
+
+            boolean isOfdpaHardware = TroubleshootUtils.hardwareOfdpaMap
+                    .getOrDefault(driverService.getDriver(in.deviceId()).name(), false);
+
+            //if the flow entry on a table is null and we are on hardware we treat as table miss, with few exceptions
+            if (flowEntry == null && isOfdpaHardware) {
+                log.debug("Ofdpa Hw setup, no flow rule means table miss");
+
+                if (((IndexTableId) tableId).id() == 27) {
+                    //Apparently a miss but Table 27 on OFDPA is a fixed table
+                    packet = handleOfdpa27FixedTable(trace, packet);
+                }
+
+                //Finding next table to go In case of miss
+                nextTableIdEntry = findNextTableIdEntry(in.deviceId(), ((IndexTableId) tableId).id());
+                log.debug("Next table id entry {}", nextTableIdEntry);
+
+                //FIXME find better solution that enable granularity greater than 0 or all rules
+                //(another possibility is max tableId)
+                if (nextTableIdEntry == null && flows.size() == 0) {
+                    trace.addResultMessage("No matching flow rules for device " + in.deviceId() + ". Aborting");
+                    computePath(completePath, trace, null);
+                    trace.setSuccess(false);
+                    return trace;
+
+                } else if (nextTableIdEntry == null) {
+                    //Means that no more flow rules are present
+                    output = true;
+
+                } else if (((IndexTableId) tableId).id() == 20) {
+                    //if the table is 20 OFDPA skips to table 50
+                    log.debug("A miss on Table 20 on OFDPA means that we skip directly to table 50");
+                    tableId = IndexTableId.of(50);
+
+                } else if (((IndexTableId) tableId).id() == 40) {
+                    //if the table is 40 OFDPA skips to table 60
+                    log.debug("A miss on Table 40 on OFDPA means that we skip directly to table 60");
+                    tableId = IndexTableId.of(60);
+                } else {
+                    tableId = nextTableIdEntry.table();
+                }
+
+            } else if (flowEntry == null) {
+                trace.addResultMessage("Packet has no match on table " + tableId + " in device " +
+                        in.deviceId() + ". Dropping");
+                computePath(completePath, trace, null);
+                trace.setSuccess(false);
+                return trace;
+            } else {
+
+                //IF the table has a transition
+                if (flowEntry.treatment().tableTransition() != null) {
+                    //update the next table we transitions to
+                    tableId = IndexTableId.of(flowEntry.treatment().tableTransition().tableId());
+                    log.debug("Flow Entry has transition to table Id {}", tableId);
+                    flows.add(flowEntry);
+                } else {
+                    //table has no transition so it means that it's an output rule if on the last table
+                    log.debug("Flow Entry has no transition to table, treating as last rule {}", flowEntry);
+                    flows.add(flowEntry);
+                    outputFlows.add(flowEntry);
+                    output = true;
+                }
+                //update the packet according to the immediate actions of this flow rule.
+                packet = updatePacket(packet, flowEntry.treatment().immediate()).build();
+
+                //save the deferred rules for later
+                deferredInstructions.addAll(flowEntry.treatment().deferred());
+
+                //If the flow requires to clear deferred actions we do so for all the ones we encountered.
+                if (flowEntry.treatment().clearedDeferred()) {
+                    deferredInstructions.clear();
+                }
+
+                //On table 10 OFDPA needs two rules to apply the vlan if none and then to transition to the next table.
+                if (needsSecondTable10Flow(flowEntry, isOfdpaHardware)) {
+
+                    //Let's get the packet vlanId instruction
+                    VlanIdCriterion packetVlanIdCriterion =
+                            (VlanIdCriterion) packet.getCriterion(Criterion.Type.VLAN_VID);
+
+                    //Let's get the flow entry vlan mod instructions
+                    ModVlanIdInstruction entryModVlanIdInstruction = (ModVlanIdInstruction) flowEntry.treatment()
+                            .immediate().stream()
+                            .filter(instruction -> instruction instanceof ModVlanIdInstruction)
+                            .findFirst().orElse(null);
+
+                    //If the entry modVlan is not null we need to make sure that the packet has been updated and there
+                    // is a flow rule that matches on same criteria and with updated vlanId
+                    if (entryModVlanIdInstruction != null) {
+
+                        FlowEntry secondVlanFlow = getSecondFlowEntryOnTable10(packet, in,
+                                packetVlanIdCriterion, entryModVlanIdInstruction);
+
+                        //We found the flow that we expected
+                        if (secondVlanFlow != null) {
+                            flows.add(secondVlanFlow);
+                        } else {
+                            trace.addResultMessage("Missing forwarding rule for tagged packet on " + in);
+                            computePath(completePath, trace, null);
+                            return trace;
+                        }
+                    }
+
+                }
+
+            }
+        }
+
+        //Creating a modifiable builder for the output packet
+        Builder builder = DefaultTrafficSelector.builder();
+        packet.criteria().forEach(builder::add);
+
+        //Adding all the flows to the trace
+        trace.addFlowsForDevice(in.deviceId(), ImmutableList.copyOf(flows));
+
+        List<PortNumber> outputPorts = new ArrayList<>();
+        List<FlowEntry> outputFlowEntries = handleFlows(trace, packet, in, outputFlows, builder, outputPorts);
+
+
+        log.debug("Handling Groups");
+        //Analyze Groups
+        List<Group> groups = new ArrayList<>();
+
+        Collection<FlowEntry> nonOutputFlows = flows;
+        nonOutputFlows.removeAll(outputFlowEntries);
+
+        //Handling groups pointed at by immediate instructions
+        for (FlowEntry entry : flows) {
+            getGroupsFromInstructions(trace, groups, entry.treatment().immediate(),
+                    entry.deviceId(), builder, outputPorts, in, completePath);
+        }
+
+        //If we have deferred instructions at this point we handle them.
+        if (deferredInstructions.size() > 0) {
+            builder = handleDeferredActions(trace, packet, in, deferredInstructions, outputPorts, groups, completePath);
+
+        }
+        packet = builder.build();
+
+        log.debug("Output Packet {}", packet);
+        return trace;
+    }
+
+    private List<FlowEntry> handleFlows(StaticPacketTrace trace, TrafficSelector packet, ConnectPoint in,
+                                        List<FlowEntry> outputFlows, Builder builder, List<PortNumber> outputPorts) {
+        //TODO optimization
+        //outputFlows contains also last rule of device, so we need filtering for OUTPUT instructions.
+        List<FlowEntry> outputFlowEntries = outputFlows.stream().filter(flow -> flow.treatment()
+                .allInstructions().stream().filter(instruction -> instruction.type()
+                        .equals(Instruction.Type.OUTPUT)).count() > 0).collect(Collectors.toList());
+
+        if (outputFlowEntries.size() > 1) {
+            trace.addResultMessage("More than one flow rule with OUTPUT instruction");
+            log.warn("There cannot be more than one flow entry with OUTPUT instruction for {}", packet);
+        }
+
+        if (outputFlowEntries.size() == 1) {
+
+            OutputInstruction outputInstruction = (OutputInstruction) outputFlowEntries.get(0).treatment()
+                    .allInstructions().stream()
+                    .filter(instruction -> {
+                        return instruction.type().equals(Instruction.Type.OUTPUT);
+                    }).findFirst().get();
+
+            //FIXME using GroupsInDevice for output even if flows.
+            buildOutputFromDevice(trace, in, builder, outputPorts, outputInstruction, ImmutableList.of());
+
+        }
+        return outputFlowEntries;
+    }
+
+    private boolean multipleRoutes(StaticPacketTrace trace) {
+        boolean multipleRoutes = false;
+        IPCriterion ipCriterion = ((IPCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.IPV4_DST));
+        IpAddress ip = null;
+        if (ipCriterion != null) {
+            ip = ipCriterion.ip().address();
+        } else if (trace.getInitialPacket().getCriterion(Criterion.Type.IPV6_DST) != null) {
+            ip = ((IPCriterion) trace.getInitialPacket().getCriterion(Criterion.Type.IPV6_DST)).ip().address();
+        }
+        if (ip != null) {
+            Optional<ResolvedRoute> optionalRoute = routeService.longestPrefixLookup(ip);
+            if (optionalRoute.isPresent()) {
+                ResolvedRoute route = optionalRoute.get();
+                route.prefix();
+                multipleRoutes = routeService.getAllResolvedRoutes(route.prefix()).size() > 1;
+            }
+        }
+        return multipleRoutes;
+    }
+
+    /**
+     * Handles the specific case where the Input is the controller.
+     * Note that the in port is used as a convenience to store the port of the controller even if the packet in
+     * from a controller should not have a physical input port. The in port from the Controller is used to make sure
+     * the flood to all active physical ports of the device.
+     *
+     * @param trace the trace
+     * @param in    the controller port
+     * @return the augmented trace.
+     */
+    private StaticPacketTrace inputFromController(StaticPacketTrace trace, ConnectPoint in) {
+        EthTypeCriterion ethTypeCriterion = (EthTypeCriterion) trace.getInitialPacket()
+                .getCriterion(Criterion.Type.ETH_TYPE);
+        //If the packet is LLDP or BDDP we flood it on all active ports of the switch.
+        if (ethTypeCriterion != null && (ethTypeCriterion.ethType().equals(EtherType.LLDP.ethType())
+                || ethTypeCriterion.ethType().equals(EtherType.BDDP.ethType()))) {
+            //get the active ports
+            List<Port> enabledPorts = deviceService.getPorts(in.deviceId()).stream()
+                    .filter(Port::isEnabled)
+                    .collect(Collectors.toList());
+            //build an output from each one
+            enabledPorts.forEach(port -> {
+                GroupsInDevice output = new GroupsInDevice(new ConnectPoint(port.element().id(), port.number()),
+                        ImmutableList.of(), trace.getInitialPacket());
+                trace.addGroupOutputPath(in.deviceId(), output);
+            });
+            return trace;
+        }
+        return null;
+    }
+
+    private boolean needsSecondTable10Flow(FlowEntry flowEntry, boolean isOfdpaHardware) {
+        return isOfdpaHardware && flowEntry.table().equals(IndexTableId.of(10))
+                && flowEntry.selector().getCriterion(Criterion.Type.VLAN_VID) != null
+                && ((VlanIdCriterion) flowEntry.selector().getCriterion(Criterion.Type.VLAN_VID))
+                .vlanId().equals(VlanId.NONE);
+    }
+
+    /**
+     * Method that finds a flow rule on table 10 that matches the packet and the VLAN of the already
+     * found rule on table 10. This is because OFDPA needs two rules on table 10, first to apply the rule,
+     * second to transition to following table
+     *
+     * @param packet                    the incoming packet
+     * @param in                        the input connect point
+     * @param packetVlanIdCriterion     the vlan criterion from the packet
+     * @param entryModVlanIdInstruction the entry vlan instruction
+     * @return the second flow entry that matched
+     */
+    private FlowEntry getSecondFlowEntryOnTable10(TrafficSelector packet, ConnectPoint in,
+                                                  VlanIdCriterion packetVlanIdCriterion,
+                                                  ModVlanIdInstruction entryModVlanIdInstruction) {
+        FlowEntry secondVlanFlow = null;
+        //Check the packet has been update from the first rule.
+        if (packetVlanIdCriterion.vlanId().equals(entryModVlanIdInstruction.vlanId())) {
+            //find a rule on the same table that matches the vlan and
+            // also all the other elements of the flow such as input port
+            secondVlanFlow = Lists.newArrayList(flowRuleService.getFlowEntriesByState(in.deviceId(),
+                    FlowEntry.FlowEntryState.ADDED).iterator())
+                    .stream()
+                    .filter(entry -> {
+                        return entry.table().equals(IndexTableId.of(10));
+                    })
+                    .filter(entry -> {
+                        VlanIdCriterion criterion = (VlanIdCriterion) entry.selector()
+                                .getCriterion(Criterion.Type.VLAN_VID);
+                        return criterion != null && match(packet, entry)
+                                && criterion.vlanId().equals(entryModVlanIdInstruction.vlanId());
+                    }).findFirst().orElse(null);
+
+        }
+        return secondVlanFlow;
+    }
+
+
+    /**
+     * Handles table 27 in Ofpda which is a fixed table not visible to any controller that handles Mpls Labels.
+     *
+     * @param packet the incoming packet
+     * @return the updated packet
+     */
+    private TrafficSelector handleOfdpa27FixedTable(StaticPacketTrace trace, TrafficSelector packet) {
+        log.debug("Handling table 27 on OFDPA, removing mpls ETH Type and change mpls label");
+        Criterion mplsCriterion = packet.getCriterion(Criterion.Type.ETH_TYPE);
+        ImmutableList.Builder<Instruction> builder = ImmutableList.builder();
+
+        //If the pakcet comes in with the expected elements we update it as per OFDPA spec.
+        if (mplsCriterion != null && ((EthTypeCriterion) mplsCriterion).ethType()
+                .equals(EtherType.MPLS_UNICAST.ethType())) {
+            //TODO update with parsing with eth MPLS pop Instruction for treating label an bos
+            Instruction ethInstruction = Instructions.popMpls(((EthTypeCriterion) trace.getInitialPacket()
+                    .getCriterion(Criterion.Type.ETH_TYPE)).ethType());
+            //FIXME what do we use as L3_Unicast mpls Label ?
+            //translateInstruction(builder, ethInstruction);
+            builder.add(ethInstruction);
+        }
+        packet = updatePacket(packet, builder.build()).build();
+        return packet;
+    }
+
+    /**
+     * Finds the flow entry with the minimun next table Id.
+     *
+     * @param deviceId  the device to search
+     * @param currentId the current id. the search will use this as minimum
+     * @return the flow entry with the minimum table Id after the given one.
+     */
+    private FlowEntry findNextTableIdEntry(DeviceId deviceId, int currentId) {
+
+        final Comparator<FlowEntry> comparator = Comparator.comparing((FlowEntry f) -> ((IndexTableId) f.table()).id());
+
+        return Lists.newArrayList(flowRuleService.getFlowEntriesByState(deviceId, FlowEntry.FlowEntryState.ADDED)
+                .iterator()).stream()
+                .filter(f -> ((IndexTableId) f.table()).id() > currentId).min(comparator).orElse(null);
+    }
+
+    private Builder handleDeferredActions(StaticPacketTrace trace, TrafficSelector packet,
+                                          ConnectPoint in, List<Instruction> deferredInstructions,
+                                          List<PortNumber> outputPorts, List<Group> groups,
+                                          List<ConnectPoint> completePath) {
+
+        //Update the packet with the deferred instructions
+        Builder builder = updatePacket(packet, deferredInstructions);
+
+        //Gather any output instructions from the deferred instruction
+        List<Instruction> outputFlowInstruction = deferredInstructions.stream().filter(instruction -> {
+            return instruction.type().equals(Instruction.Type.OUTPUT);
+        }).collect(Collectors.toList());
+
+        //We are considering deferred instructions from flows, there can only be one output.
+        if (outputFlowInstruction.size() > 1) {
+            trace.addResultMessage("More than one flow rule with OUTPUT instruction");
+            log.warn("There cannot be more than one flow entry with OUTPUT instruction for {}", packet);
+        }
+        //If there is one output let's go through that
+        if (outputFlowInstruction.size() == 1) {
+            buildOutputFromDevice(trace, in, builder, outputPorts, (OutputInstruction) outputFlowInstruction.get(0),
+                    ImmutableList.of());
+        }
+        //If there is no output let's see if there any deferred instruction point to groups.
+        if (outputFlowInstruction.size() == 0) {
+            getGroupsFromInstructions(trace, groups, deferredInstructions,
+                    in.deviceId(), builder, outputPorts, in, completePath);
+        }
+        return builder;
+    }
+
+    /**
+     * Gets group information from instructions.
+     *
+     * @param trace           the trace we are building
+     * @param groupsForDevice the set of groups for this device
+     * @param instructions    the set of instructions we are searching for groups.
+     * @param deviceId        the device we are considering
+     * @param builder         the builder of the input packet
+     * @param outputPorts     the output ports for that packet
+     */
+    private void getGroupsFromInstructions(StaticPacketTrace trace, List<Group> groupsForDevice,
+                                           List<Instruction> instructions, DeviceId deviceId,
+                                           Builder builder, List<PortNumber> outputPorts,
+                                           ConnectPoint in, List<ConnectPoint> completePath) {
+        List<Instruction> groupInstructionlist = new ArrayList<>();
+        for (Instruction instruction : instructions) {
+            log.debug("Considering Instruction {}", instruction);
+            //if the instruction is not group we need to update the packet or add the output
+            //to the possible outputs for this packet
+            if (!instruction.type().equals(Instruction.Type.GROUP)) {
+                //if the instruction is not group we need to update the packet or add the output
+                //to the possible outputs for this packet
+                if (instruction.type().equals(Instruction.Type.OUTPUT)) {
+                    buildOutputFromDevice(trace, in, builder, outputPorts,
+                            (OutputInstruction) instruction, ImmutableList.copyOf(groupsForDevice));
+                    //clearing the groups because we start from the top.
+                    groupsForDevice.clear();
+                } else {
+                    builder = translateInstruction(builder, instruction);
+                }
+            } else {
+                //if the instuction is pointing to a group we need to get the group
+                groupInstructionlist.add(instruction);
+            }
+        }
+        //handle all the internal instructions pointing to a group.
+        for (Instruction instr : groupInstructionlist) {
+            GroupInstruction groupInstruction = (GroupInstruction) instr;
+            Group group = Lists.newArrayList(groupService.getGroups(deviceId)).stream().filter(groupInternal -> {
+                return groupInternal.id().equals(groupInstruction.groupId());
+            }).findAny().orElse(null);
+            if (group == null) {
+                trace.addResultMessage("Null group for Instruction " + instr);
+                trace.setSuccess(false);
+                break;
+            }
+            if (group.buckets().buckets().size() == 0) {
+                trace.addResultMessage("Group " + group.id() + " has no buckets");
+                trace.setSuccess(false);
+                computePath(completePath, trace, null);
+                break;
+            }
+
+            //Cycle in each of the group's buckets and add them to the groups for this Device.
+            for (GroupBucket bucket : group.buckets().buckets()) {
+
+                //add the group to the traversed groups
+                if (!groupsForDevice.contains(group)) {
+                    groupsForDevice.add(group);
+                }
+
+                getGroupsFromInstructions(trace, groupsForDevice, bucket.treatment().allInstructions(),
+                        deviceId, builder, outputPorts, in, completePath);
+            }
+        }
+    }
+
+    /**
+     * Check if the output is the input port, if so adds a dop result message, otherwise builds
+     * a possible output from this device.
+     *
+     * @param trace             the trace
+     * @param in                the input connect point
+     * @param builder           the packet builder
+     * @param outputPorts       the list of output ports for this device
+     * @param outputInstruction the output instruction
+     * @param groupsForDevice   the groups we output from
+     */
+    private void buildOutputFromDevice(StaticPacketTrace trace, ConnectPoint in, Builder builder,
+                                       List<PortNumber> outputPorts, OutputInstruction outputInstruction,
+                                       List<Group> groupsForDevice) {
+        ConnectPoint output = new ConnectPoint(in.deviceId(), outputInstruction.port());
+
+        outputPorts.add(outputInstruction.port());
+
+        GroupsInDevice device = new GroupsInDevice(output, groupsForDevice, builder.build());
+        if (trace.getGroupOuputs(output.deviceId()) != null
+                && trace.getGroupOuputs(output.deviceId()).contains(device)) {
+            return;
+        }
+        trace.addGroupOutputPath(in.deviceId(),
+                new GroupsInDevice(output, groupsForDevice, builder.build()));
+    }
+
+    /**
+     * Applies all give instructions to the input packet.
+     *
+     * @param packet       the input packet
+     * @param instructions the set of instructions
+     * @return the packet with the applied instructions
+     */
+    private Builder updatePacket(TrafficSelector packet, List<Instruction> instructions) {
+        Builder newSelector = DefaultTrafficSelector.builder();
+        packet.criteria().forEach(newSelector::add);
+        //FIXME optimize
+        for (Instruction instruction : instructions) {
+            newSelector = translateInstruction(newSelector, instruction);
+        }
+        return newSelector;
+    }
+
+    /**
+     * Applies an instruction to the packet in the form of a selector.
+     *
+     * @param newSelector the packet selector
+     * @param instruction the instruction to be translated
+     * @return the new selector with the applied instruction
+     */
+    private Builder translateInstruction(Builder newSelector, Instruction instruction) {
+        log.debug("Translating instruction {}", instruction);
+        log.debug("New Selector {}", newSelector.build());
+        //TODO add as required
+        Criterion criterion = null;
+        switch (instruction.type()) {
+            case L2MODIFICATION:
+                L2ModificationInstruction l2Instruction = (L2ModificationInstruction) instruction;
+                switch (l2Instruction.subtype()) {
+                    case VLAN_ID:
+                        ModVlanIdInstruction vlanIdInstruction =
+                                (ModVlanIdInstruction) instruction;
+                        VlanId id = vlanIdInstruction.vlanId();
+                        criterion = Criteria.matchVlanId(id);
+                        break;
+                    case VLAN_POP:
+                        criterion = Criteria.matchVlanId(VlanId.NONE);
+                        break;
+                    case MPLS_PUSH:
+                        ModMplsHeaderInstruction mplsEthInstruction =
+                                (ModMplsHeaderInstruction) instruction;
+                        criterion = Criteria.matchEthType(mplsEthInstruction.ethernetType().toShort());
+                        break;
+                    case MPLS_POP:
+                        ModMplsHeaderInstruction mplsPopInstruction =
+                                (ModMplsHeaderInstruction) instruction;
+                        criterion = Criteria.matchEthType(mplsPopInstruction.ethernetType().toShort());
+
+                        //When popping MPLS we remove label and BOS
+                        TrafficSelector temporaryPacket = newSelector.build();
+                        if (temporaryPacket.getCriterion(Criterion.Type.MPLS_LABEL) != null) {
+                            Builder noMplsSelector = DefaultTrafficSelector.builder();
+                            temporaryPacket.criteria().stream().filter(c -> {
+                                return !c.type().equals(Criterion.Type.MPLS_LABEL) &&
+                                        !c.type().equals(Criterion.Type.MPLS_BOS);
+                            }).forEach(noMplsSelector::add);
+                            newSelector = noMplsSelector;
+                        }
+
+                        break;
+                    case MPLS_LABEL:
+                        ModMplsLabelInstruction mplsLabelInstruction =
+                                (ModMplsLabelInstruction) instruction;
+                        criterion = Criteria.matchMplsLabel(mplsLabelInstruction.label());
+                        newSelector.matchMplsBos(true);
+                        break;
+                    case ETH_DST:
+                        ModEtherInstruction modEtherDstInstruction =
+                                (ModEtherInstruction) instruction;
+                        criterion = Criteria.matchEthDst(modEtherDstInstruction.mac());
+                        break;
+                    case ETH_SRC:
+                        ModEtherInstruction modEtherSrcInstruction =
+                                (ModEtherInstruction) instruction;
+                        criterion = Criteria.matchEthSrc(modEtherSrcInstruction.mac());
+                        break;
+                    default:
+                        log.debug("Unsupported L2 Instruction");
+                        break;
+                }
+                break;
+            default:
+                log.debug("Unsupported Instruction");
+                break;
+        }
+        if (criterion != null) {
+            log.debug("Adding criterion {}", criterion);
+            newSelector.add(criterion);
+        }
+        return newSelector;
+    }
+
+    /**
+     * Finds the rule in the device that mathces the input packet and has the highest priority.
+     *
+     * @param packet  the input packet
+     * @param in      the connect point the packet comes in from
+     * @param tableId the table to search
+     * @return the flow entry
+     */
+    private FlowEntry matchHighestPriority(TrafficSelector packet, ConnectPoint in, TableId tableId) {
+        //Computing the possible match rules.
+        final Comparator<FlowEntry> comparator = Comparator.comparing(FlowRule::priority);
+        return Lists.newArrayList(flowRuleService.getFlowEntriesByState(in.deviceId(), FlowEntry.FlowEntryState.ADDED)
+                .iterator()).stream()
+                .filter(flowEntry -> {
+                    return flowEntry.table().equals(tableId);
+                })
+                .filter(flowEntry -> {
+                    return match(packet, flowEntry);
+                }).max(comparator).orElse(null);
+    }
+
+    /**
+     * Matches the packet with the given flow entry.
+     *
+     * @param packet    the packet to match
+     * @param flowEntry the flow entry to match the packet against
+     * @return true if the packet matches the flow.
+     */
+    private boolean match(TrafficSelector packet, FlowEntry flowEntry) {
+        return flowEntry.selector().criteria().stream().allMatch(criterion -> {
+            Criterion.Type type = criterion.type();
+            //If the criterion has IP we need to do LPM to establish matching.
+            if (type.equals(Criterion.Type.IPV4_SRC) || type.equals(Criterion.Type.IPV4_DST) ||
+                    type.equals(Criterion.Type.IPV6_SRC) || type.equals(Criterion.Type.IPV6_DST)) {
+                return matchIp(packet, (IPCriterion) criterion);
+                //we check that the packet contains the criterion provided by the flow rule.
+            } else if (type.equals(Criterion.Type.ETH_SRC_MASKED)) {
+                return matchMac(packet, (EthCriterion) criterion, false);
+            } else if (type.equals(Criterion.Type.ETH_DST_MASKED)) {
+                return matchMac(packet, (EthCriterion) criterion, true);
+            } else {
+                return packet.criteria().contains(criterion);
+            }
+        });
+    }
+
+    /**
+     * Checks if the packet has an dst or src IP and if that IP matches the subnet of the ip criterion.
+     *
+     * @param packet    the incoming packet
+     * @param criterion the criterion to match
+     * @return true if match
+     */
+    private boolean matchIp(TrafficSelector packet, IPCriterion criterion) {
+        IPCriterion matchCriterion = (IPCriterion) packet.getCriterion(criterion.type());
+        //if the packet does not have an IPv4 or IPv6 criterion we return true
+        if (matchCriterion == null) {
+            return false;
+        }
+        try {
+            log.debug("Checking if {} is under {}", matchCriterion.ip(), criterion.ip());
+            Subnet subnet = Subnet.createInstance(criterion.ip().toString());
+            return subnet.isInSubnet(matchCriterion.ip().address().toInetAddress());
+        } catch (UnknownHostException e) {
+            return false;
+        }
+    }
+
+    /**
+     * Checks if the packet has a dst or src MAC and if that Mac matches the mask of the mac criterion.
+     *
+     * @param packet       the incoming packet
+     * @param hitCriterion the criterion to match
+     * @param dst          true if we are checking DST MAC
+     * @return true if match
+     */
+    private boolean matchMac(TrafficSelector packet, EthCriterion hitCriterion, boolean dst) {
+        //Packet can have only one EthCriterion
+        EthCriterion matchCriterion;
+        if (dst) {
+            matchCriterion = (EthCriterion) packet.criteria().stream().filter(criterion1 -> {
+                return criterion1.type().equals(Criterion.Type.ETH_DST_MASKED) ||
+                        criterion1.type().equals(Criterion.Type.ETH_DST);
+            }).findFirst().orElse(null);
+        } else {
+            matchCriterion = (EthCriterion) packet.criteria().stream().filter(criterion1 -> {
+                return criterion1.type().equals(Criterion.Type.ETH_SRC_MASKED) ||
+                        criterion1.type().equals(Criterion.Type.ETH_SRC);
+            }).findFirst().orElse(null);
+        }
+        //if the packet does not have an ETH criterion we return true
+        if (matchCriterion == null) {
+            return true;
+        }
+        log.debug("Checking if {} is under {}/{}", matchCriterion.mac(), hitCriterion.mac(), hitCriterion.mask());
+        return compareMac(matchCriterion.mac(), hitCriterion.mac(), hitCriterion.mask());
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/impl/TroubleshootUtils.java b/apps/t3/app/src/main/java/org/onosproject/t3/impl/TroubleshootUtils.java
new file mode 100644
index 0000000..57758b3
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/impl/TroubleshootUtils.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.t3.impl;
+
+import com.google.common.collect.ImmutableMap;
+import org.onlab.packet.MacAddress;
+
+import java.util.Map;
+
+/**
+ * Utility class for the troubleshooting tool.
+ */
+final class TroubleshootUtils {
+
+    private TroubleshootUtils() {
+        //Banning construction
+    }
+
+    /**
+     * Map defining if a specific driver is for a HW switch.
+     */
+    //Done with builder() instead of of() for clarity
+    static Map<String, Boolean> hardwareOfdpaMap = ImmutableMap.<String, Boolean>builder()
+            .put("ofdpa", true)
+            .put("ofdpa3", true)
+            .put("qmx-ofdpa3", true)
+            .put("as7712-32x-premium", true)
+            .put("as5912-54x-premium", true)
+            .put("as5916-54x-premium", true)
+            .put("accton-ofdpa3", true)
+            .put("znyx-ofdpa", true)
+            .build();
+
+    /**
+     * Checks if the Mac Address is inside a range between the min MAC and the mask.
+     * @param macAddress the MAC address to check
+     * @param minAddr the min MAC address
+     * @param maskAddr the mask
+     * @return true if in range, false otherwise.
+     */
+    static boolean compareMac(MacAddress macAddress, MacAddress minAddr, MacAddress maskAddr) {
+        byte[] mac = macAddress.toBytes();
+        byte[] min = minAddr.toBytes();
+        byte[] mask = maskAddr.toBytes();
+        boolean inRange = true;
+
+        int i = 0;
+
+        //if mask is 00 stop
+        while (inRange && i < mask.length && (mask[i] & 0xFF) != 0) {
+            int ibmac = mac[i] & 0xFF;
+            int ibmin = min[i] & 0xFF;
+            int ibmask = mask[i] & 0xFF;
+            if (ibmask == 255) {
+                inRange = ibmac == ibmin;
+            } else if (ibmac < ibmin || ibmac >= ibmask) {
+                inRange = false;
+                break;
+            }
+            i++;
+        }
+
+        return inRange;
+    }
+}
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/impl/package-info.java b/apps/t3/app/src/main/java/org/onosproject/t3/impl/package-info.java
new file mode 100644
index 0000000..dfcc294
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/impl/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Troubleshooting toolkit for trellis fabrics implementation.
+ */
+package org.onosproject.t3.impl;
diff --git a/apps/t3/app/src/main/java/org/onosproject/t3/package-info.java b/apps/t3/app/src/main/java/org/onosproject/t3/package-info.java
new file mode 100644
index 0000000..2dae6e2
--- /dev/null
+++ b/apps/t3/app/src/main/java/org/onosproject/t3/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Troubleshooting toolkit for trellis fabrics.
+ */
+package org.onosproject.t3;
diff --git a/apps/t3/app/src/main/resources/OSGI-INF/blueprint/shell-config.xml b/apps/t3/app/src/main/resources/OSGI-INF/blueprint/shell-config.xml
new file mode 100644
index 0000000..48df042
--- /dev/null
+++ b/apps/t3/app/src/main/resources/OSGI-INF/blueprint/shell-config.xml
@@ -0,0 +1,47 @@
+<!--
+  ~ Copyright 2017-present Open Networking Foundation
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0">
+
+    <command-bundle xmlns="http://karaf.apache.org/xmlns/shell/v1.1.0">
+        <command>
+            <action class="org.onosproject.t3.cli.TroubleshootTraceCommand"/>
+        </command>
+        <command>
+            <action class="org.onosproject.t3.cli.TroubleshootSimpleTraceCommand"/>
+            <completers>
+                <ref component-id="hostIdCompleter"/>
+            </completers>
+            <optional-completers>
+                <entry key="-et" value-ref="ethTypeCompleter"/>
+            </optional-completers>
+        </command>
+        <command>
+            <action class="org.onosproject.t3.cli.TroubleshootPingAllCommand"/>
+            <optional-completers>
+                <entry key="-et" value-ref="ethTypeCompleter"/>
+            </optional-completers>
+        </command>
+        <command>
+            <action class="org.onosproject.t3.cli.TroubleshootMcastCommand"/>
+        </command>
+    </command-bundle>
+
+    <bean id="hostIdCompleter" class="org.onosproject.cli.net.HostIdCompleter"/>
+    <bean id="ethTypeCompleter" class="org.onosproject.cli.net.EthTypeCompleter"/>
+
+</blueprint>
+
+
diff --git a/apps/t3/app/src/test/java/org/onosproject/t3/impl/T3TestObjects.java b/apps/t3/app/src/test/java/org/onosproject/t3/impl/T3TestObjects.java
new file mode 100644
index 0000000..8501400
--- /dev/null
+++ b/apps/t3/app/src/test/java/org/onosproject/t3/impl/T3TestObjects.java
@@ -0,0 +1,760 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.t3.impl;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.onlab.packet.EthType;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.MplsLabel;
+import org.onlab.packet.VlanId;
+import org.onosproject.core.DefaultApplicationId;
+import org.onosproject.core.GroupId;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.net.DefaultHost;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.HostId;
+import org.onosproject.net.HostLocation;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.flow.DefaultFlowEntry;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.FlowEntry;
+import org.onosproject.net.flow.FlowRule;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.group.DefaultGroup;
+import org.onosproject.net.group.DefaultGroupBucket;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupBucket;
+import org.onosproject.net.group.GroupBuckets;
+import org.onosproject.net.provider.ProviderId;
+
+/**
+ * Helper class for objects related to the Troubleshoot Manager Test.
+ */
+final class T3TestObjects {
+
+    private T3TestObjects() {
+        //banning construction
+    }
+
+    private static final String HOST_ONE_MAC = "00:00:00:00:00:01";
+    private static final String HOST_TWO_MAC = "00:00:00:00:00:02";
+    private static final String HOST_ONE_VLAN = "None";
+    private static final String HOST_TWO_VLAN = "None";
+    private static final String HOST_ONE = HOST_ONE_MAC + "/" + HOST_ONE_VLAN;
+    private static final String HOST_TWO = HOST_TWO_MAC + "/" + HOST_TWO_VLAN;
+    private static final String HOST_DUAL_HOMED_MAC = "00:00:00:00:00:03";
+    private static final String HOST_DUAL_HOMED_VLAN = "None";
+    private static final String HOST_DUAL_HOMED = HOST_DUAL_HOMED_MAC + "/" + HOST_DUAL_HOMED_VLAN;
+
+    //offline device
+    static final DeviceId OFFLINE_DEVICE = DeviceId.deviceId("offlineDevice");
+
+    //Single Flow Test
+    static final DeviceId SINGLE_FLOW_DEVICE = DeviceId.deviceId("SingleFlowDevice");
+    private static final TrafficSelector SINGLE_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .build();
+
+    private static final TrafficTreatment OUTPUT_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setOutput(PortNumber.portNumber(2)).build();
+    private static final FlowRule SINGLE_FLOW = DefaultFlowEntry.builder().forDevice(SINGLE_FLOW_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(SINGLE_FLOW_SELECTOR)
+            .withTreatment(OUTPUT_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry SINGLE_FLOW_ENTRY = new DefaultFlowEntry(SINGLE_FLOW);
+
+    static final ConnectPoint SINGLE_FLOW_IN_CP = ConnectPoint.deviceConnectPoint(SINGLE_FLOW_DEVICE + "/" + 1);
+
+    static final ConnectPoint SINGLE_FLOW_OUT_CP = ConnectPoint.deviceConnectPoint(SINGLE_FLOW_DEVICE + "/" + 2);
+
+    //same output as input
+    static final DeviceId SAME_OUTPUT_FLOW_DEVICE = DeviceId.deviceId("sameOutputDevice");
+
+    private static final TrafficTreatment SAME_OUTPUT_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setOutput(PortNumber.portNumber(1)).build();
+    private static final FlowRule SAME_OUTPUT_FLOW = DefaultFlowEntry.builder().forDevice(SAME_OUTPUT_FLOW_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(SINGLE_FLOW_SELECTOR)
+            .withTreatment(SAME_OUTPUT_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry SAME_OUTPUT_FLOW_ENTRY = new DefaultFlowEntry(SAME_OUTPUT_FLOW);
+
+    static final ConnectPoint SAME_OUTPUT_FLOW_CP = ConnectPoint.deviceConnectPoint(SAME_OUTPUT_FLOW_DEVICE + "/" + 1);
+
+    //ARP
+    static final DeviceId ARP_FLOW_DEVICE = DeviceId.deviceId("ArpDevice");
+
+    private static final TrafficSelector ARP_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+            .build();
+
+    private static final TrafficTreatment ARP_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setOutput(PortNumber.CONTROLLER).build();
+    private static final FlowRule ARP_FLOW = DefaultFlowEntry.builder().forDevice(ARP_FLOW_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(ARP_FLOW_SELECTOR)
+            .withTreatment(ARP_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry ARP_FLOW_ENTRY = new DefaultFlowEntry(ARP_FLOW);
+
+    static final ConnectPoint ARP_FLOW_CP = ConnectPoint.deviceConnectPoint(ARP_FLOW_DEVICE + "/" + 1);
+
+
+    //Dual Flow Test
+    static final DeviceId DUAL_FLOW_DEVICE = DeviceId.deviceId("DualFlowDevice");
+    private static final TrafficTreatment TRANSITION_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setVlanId(VlanId.vlanId((short) 100))
+            .transition(10)
+            .build();
+    private static final TrafficSelector VLAN_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchVlanId(VlanId.vlanId((short) 100))
+            .build();
+    private static final FlowRule FIRST_FLOW = DefaultFlowEntry.builder().forDevice(DUAL_FLOW_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(SINGLE_FLOW_SELECTOR)
+            .withTreatment(TRANSITION_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry FIRST_FLOW_ENTRY = new DefaultFlowEntry(FIRST_FLOW);
+    private static final FlowRule SECOND_FLOW = DefaultFlowEntry.builder().forDevice(DUAL_FLOW_DEVICE)
+            .forTable(10)
+            .withPriority(100)
+            .withSelector(VLAN_FLOW_SELECTOR)
+            .withTreatment(OUTPUT_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry SECOND_FLOW_ENTRY = new DefaultFlowEntry(SECOND_FLOW);
+
+    static final ConnectPoint DUAL_FLOW_IN_CP = ConnectPoint.deviceConnectPoint(DUAL_FLOW_DEVICE + "/" + 1);
+
+    static final ConnectPoint DUAL_FLOW_OUT_CP = ConnectPoint.deviceConnectPoint(DUAL_FLOW_DEVICE + "/" + 2);
+
+    //Flow and Group Test
+    static final DeviceId GROUP_FLOW_DEVICE = DeviceId.deviceId("GroupFlowDevice");
+
+    private static final GroupId GROUP_ID = GroupId.valueOf(1);
+
+    private static final TrafficTreatment GROUP_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .pushMpls()
+            .setMpls(MplsLabel.mplsLabel(100))
+            .group(GROUP_ID)
+            .build();
+    private static final FlowRule GROUP_FLOW = DefaultFlowEntry.builder().forDevice(GROUP_FLOW_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(SINGLE_FLOW_SELECTOR)
+            .withTreatment(GROUP_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry GROUP_FLOW_ENTRY = new DefaultFlowEntry(GROUP_FLOW);
+
+    private static final TrafficTreatment OUTPUT_GROUP_TREATMENT = DefaultTrafficTreatment.builder()
+            .popMpls(EthType.EtherType.IPV4.ethType())
+            .setOutput(PortNumber.portNumber(2)).build();
+
+    private static final GroupBucket BUCKET = DefaultGroupBucket.createSelectGroupBucket(OUTPUT_GROUP_TREATMENT);
+
+    private static final GroupBuckets BUCKETS = new GroupBuckets(ImmutableList.of(BUCKET));
+
+    static final Group GROUP = new DefaultGroup(GROUP_ID, GROUP_FLOW_DEVICE, Group.Type.SELECT, BUCKETS);
+
+    static final ConnectPoint GROUP_FLOW_IN_CP = ConnectPoint.deviceConnectPoint(GROUP_FLOW_DEVICE + "/" + 1);
+
+    static final ConnectPoint GROUP_FLOW_OUT_CP = ConnectPoint.deviceConnectPoint(GROUP_FLOW_DEVICE + "/" + 2);
+
+    //topology
+
+    static final DeviceId TOPO_FLOW_DEVICE = DeviceId.deviceId("SingleFlowDevice1");
+
+    static final DeviceId TOPO_FLOW_2_DEVICE = DeviceId.deviceId("SingleFlowDevice2");
+
+    static final DeviceId TOPO_FLOW_3_DEVICE = DeviceId.deviceId("SingleFlowDevice3");
+
+    private static final TrafficSelector TOPO_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.3/32"))
+            .build();
+
+    private static final FlowRule TOPO_SINGLE_FLOW = DefaultFlowEntry.builder().forDevice(TOPO_FLOW_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(TOPO_FLOW_SELECTOR)
+            .withTreatment(OUTPUT_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    static final FlowEntry TOPO_SINGLE_FLOW_ENTRY = new DefaultFlowEntry(TOPO_SINGLE_FLOW);
+
+    static final ConnectPoint TOPO_FLOW_1_IN_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_DEVICE + "/" + 1);
+
+    static final ConnectPoint TOPO_FLOW_1_OUT_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_DEVICE + "/" + 2);
+
+    static final ConnectPoint TOPO_FLOW_2_IN_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_2_DEVICE + "/" + 1);
+
+    static final ConnectPoint TOPO_FLOW_2_OUT_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_2_DEVICE + "/" + 2);
+
+    static final ConnectPoint TOPO_FLOW_3_IN_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_3_DEVICE + "/" + 1);
+
+    static final ConnectPoint TOPO_FLOW_3_OUT_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_3_DEVICE + "/" + 2);
+
+
+    //Topology with Groups
+
+    static final DeviceId TOPO_GROUP_FLOW_DEVICE = DeviceId.deviceId("TopoGroupFlowDevice");
+
+    private static final TrafficSelector TOPO_SECOND_INPUT_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(3))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.3/32"))
+            .build();
+
+    private static final FlowRule TOPO_SECOND_INPUT_FLOW = DefaultFlowEntry.builder().forDevice(TOPO_FLOW_3_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(TOPO_SECOND_INPUT_FLOW_SELECTOR)
+            .withTreatment(OUTPUT_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    private static final TrafficTreatment OUTPUT_2_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setOutput(PortNumber.portNumber(3)).build();
+
+
+    private static final GroupId TOPO_GROUP_ID = GroupId.valueOf(1);
+
+    private static final TrafficTreatment TOPO_GROUP_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .group(TOPO_GROUP_ID)
+            .build();
+    private static final FlowRule TOPO_GROUP_FLOW = DefaultFlowEntry.builder().forDevice(TOPO_GROUP_FLOW_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(TOPO_FLOW_SELECTOR)
+            .withTreatment(TOPO_GROUP_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry TOPO_GROUP_FLOW_ENTRY = new DefaultFlowEntry(TOPO_GROUP_FLOW);
+
+    private static final GroupBucket BUCKET_2 = DefaultGroupBucket.createSelectGroupBucket(OUTPUT_2_FLOW_TREATMENT);
+
+    private static final GroupBuckets BUCKETS_MULTIPLE = new GroupBuckets(ImmutableList.of(BUCKET, BUCKET_2));
+
+    static final Group TOPO_GROUP = new DefaultGroup(TOPO_GROUP_ID, TOPO_GROUP_FLOW_DEVICE,
+            Group.Type.SELECT, BUCKETS_MULTIPLE);
+
+    static final FlowEntry TOPO_SECOND_INPUT_FLOW_ENTRY = new DefaultFlowEntry(TOPO_SECOND_INPUT_FLOW);
+
+    static final DeviceId TOPO_FLOW_4_DEVICE = DeviceId.deviceId("SingleFlowDevice4");
+
+    static final ConnectPoint TOPO_FLOW_IN_CP = ConnectPoint.deviceConnectPoint(TOPO_GROUP_FLOW_DEVICE + "/" + 1);
+
+    static final ConnectPoint TOPO_FLOW_OUT_CP_1 = ConnectPoint.deviceConnectPoint(TOPO_GROUP_FLOW_DEVICE + "/" + 2);
+
+    protected static final ConnectPoint TOPO_FLOW_OUT_CP_2 =
+            ConnectPoint.deviceConnectPoint(TOPO_GROUP_FLOW_DEVICE + "/" + 3);
+
+    static final ConnectPoint TOPO_FLOW_4_IN_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_4_DEVICE + "/" + 1);
+
+    static final ConnectPoint TOPO_FLOW_3_IN_2_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_3_DEVICE + "/" + 3);
+
+    static final ConnectPoint TOPO_FLOW_4_OUT_CP = ConnectPoint.deviceConnectPoint(TOPO_FLOW_4_DEVICE + "/" + 2);
+
+
+    //hardware
+
+    static final DeviceId HARDWARE_DEVICE = DeviceId.deviceId("HardwareDevice");
+
+    static final ConnectPoint HARDWARE_DEVICE_IN_CP = ConnectPoint.deviceConnectPoint(HARDWARE_DEVICE + "/" + 1);
+
+    static final ConnectPoint HARDWARE_DEVICE_OUT_CP = ConnectPoint.deviceConnectPoint(HARDWARE_DEVICE + "/" + 2);
+
+    private static final TrafficSelector HARDWARE_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .build();
+
+    private static final TrafficTreatment HW_TRANSITION_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .pushMpls()
+            .transition(27)
+            .build();
+
+    private static final FlowRule HARDWARE_FLOW = DefaultFlowEntry.builder().forDevice(TOPO_FLOW_3_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(HARDWARE_FLOW_SELECTOR)
+            .withTreatment(HW_TRANSITION_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    static final FlowEntry HARDWARE_FLOW_ENTRY = new DefaultFlowEntry(HARDWARE_FLOW);
+
+    private static final TrafficSelector HARDWARE_ETH_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .matchEthType(EthType.EtherType.IPV4.ethType().toShort())
+            .build();
+
+    private static final FlowRule HARDWARE_ETH_FLOW = DefaultFlowEntry.builder().forDevice(TOPO_FLOW_3_DEVICE)
+            .forTable(30)
+            .withPriority(100)
+            .withSelector(HARDWARE_ETH_FLOW_SELECTOR)
+            .withTreatment(OUTPUT_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    static final FlowEntry HARDWARE_ETH_FLOW_ENTRY = new DefaultFlowEntry(HARDWARE_ETH_FLOW);
+
+    //HW Double Rule on 10
+
+    static final DeviceId HARDWARE_DEVICE_10 = DeviceId.deviceId("HardwareDevice10");
+
+    static final ConnectPoint HARDWARE_DEVICE_10_IN_CP = ConnectPoint.deviceConnectPoint(HARDWARE_DEVICE_10 + "/" + 1);
+
+    static final ConnectPoint HARDWARE_DEVICE_10_OUT_CP = ConnectPoint.deviceConnectPoint(HARDWARE_DEVICE_10 + "/" + 2);
+
+    private static final TrafficSelector HARDWARE_10_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .matchVlanId(VlanId.NONE)
+            .build();
+
+    private static final TrafficTreatment HARDWARE_10_TRANSITION_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setVlanId(VlanId.vlanId("10"))
+            .transition(20)
+            .build();
+
+    private static final FlowRule HARDWARE_DEVICE_10_FLOW = DefaultFlowEntry.builder().forDevice(HARDWARE_DEVICE_10)
+            .forTable(10)
+            .withPriority(100)
+            .withSelector(HARDWARE_10_FLOW_SELECTOR)
+            .withTreatment(HARDWARE_10_TRANSITION_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    static final FlowEntry HARDWARE_10_FLOW_ENTRY = new DefaultFlowEntry(HARDWARE_DEVICE_10_FLOW);
+
+    private static final TrafficSelector HARDWARE_10_SECOND_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchVlanId(VlanId.vlanId("10"))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .matchEthType(EthType.EtherType.IPV4.ethType().toShort())
+            .build();
+
+    private static final FlowRule HARDWARE_10_SECOND_FLOW = DefaultFlowEntry.builder().forDevice(HARDWARE_DEVICE_10)
+            .forTable(10)
+            .withPriority(100)
+            .withSelector(HARDWARE_10_SECOND_SELECTOR)
+            .withTreatment(HARDWARE_10_TRANSITION_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    static final FlowEntry HARDWARE_10_SECOND_FLOW_ENTRY = new DefaultFlowEntry(HARDWARE_10_SECOND_FLOW);
+
+    private static final FlowRule HARDWARE_10_OUTPUT_FLOW = DefaultFlowEntry.builder().forDevice(HARDWARE_DEVICE_10)
+            .forTable(20)
+            .withPriority(100)
+            .withSelector(SINGLE_FLOW_SELECTOR)
+            .withTreatment(OUTPUT_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    static final FlowEntry HARDWARE_10_OUTPUT_FLOW_ENTRY = new DefaultFlowEntry(HARDWARE_10_OUTPUT_FLOW);
+
+    //Dual Links
+    // - (1) Device 1 (2-3) - (1-4) Device 2 (2-3) - (1-2) Device 3 (3) -
+    static final DeviceId DUAL_LINK_1 = DeviceId.deviceId("DualLink1");
+    static final DeviceId DUAL_LINK_2 = DeviceId.deviceId("DualLink2");
+    static final DeviceId DUAL_LINK_3 = DeviceId.deviceId("DualLink3");
+
+    static final ConnectPoint DUAL_LINK_1_CP_1_IN = ConnectPoint.deviceConnectPoint(DUAL_LINK_1 + "/" + 1);
+    static final ConnectPoint DUAL_LINK_1_CP_2_OUT = ConnectPoint.deviceConnectPoint(DUAL_LINK_1 + "/" + 2);
+    static final ConnectPoint DUAL_LINK_1_CP_3_OUT = ConnectPoint.deviceConnectPoint(DUAL_LINK_1 + "/" + 3);
+    static final ConnectPoint DUAL_LINK_2_CP_1_IN = ConnectPoint.deviceConnectPoint(DUAL_LINK_2 + "/" + 1);
+    static final ConnectPoint DUAL_LINK_2_CP_4_IN = ConnectPoint.deviceConnectPoint(DUAL_LINK_2 + "/" + 4);
+    static final ConnectPoint DUAL_LINK_2_CP_2_OUT = ConnectPoint.deviceConnectPoint(DUAL_LINK_2 + "/" + 2);
+    static final ConnectPoint DUAL_LINK_2_CP_3_OUT = ConnectPoint.deviceConnectPoint(DUAL_LINK_2 + "/" + 3);
+    static final ConnectPoint DUAL_LINK_3_CP_1_IN = ConnectPoint.deviceConnectPoint(DUAL_LINK_3 + "/" + 1);
+    static final ConnectPoint DUAL_LINK_3_CP_2_IN = ConnectPoint.deviceConnectPoint(DUAL_LINK_3 + "/" + 2);
+    static final ConnectPoint DUAL_LINK_3_CP_3_OUT = ConnectPoint.deviceConnectPoint(DUAL_LINK_3 + "/" + 3);
+
+    //match on port 1 and point to group for device 1 and 2
+    private static final TrafficTreatment DUAL_LINK_1_GROUP_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .pushMpls()
+            .setMpls(MplsLabel.mplsLabel(100))
+            .group(GROUP_ID)
+            .build();
+    private static final FlowRule DUAL_LINK_1_GROUP_FLOW = DefaultFlowEntry.builder().forDevice(DUAL_LINK_1)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(SINGLE_FLOW_SELECTOR)
+            .withTreatment(DUAL_LINK_1_GROUP_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry DUAL_LINK_1_GROUP_FLOW_ENTRY = new DefaultFlowEntry(DUAL_LINK_1_GROUP_FLOW);
+
+    //Match on port 4 and point to group for device 2
+    private static final TrafficSelector DUAL_LINK_2_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(4))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .build();
+
+    private static final FlowRule DUAL_LINK_2_GROUP_FLOW = DefaultFlowEntry.builder().forDevice(DUAL_LINK_2)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(DUAL_LINK_2_FLOW_SELECTOR)
+            .withTreatment(DUAL_LINK_1_GROUP_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry DUAL_LINK_2_GROUP_FLOW_ENTRY = new DefaultFlowEntry(DUAL_LINK_2_GROUP_FLOW);
+
+    //Flows for device 3 to ouput on port 3
+    private static final TrafficTreatment DUAL_LINK_1_OUTPUT_TREATMENT = DefaultTrafficTreatment.builder()
+            .popMpls(EthType.EtherType.IPV4.ethType())
+            .setOutput(PortNumber.portNumber(3)).build();
+
+    private static final TrafficSelector DUAL_LINK_3_FLOW_SELECTOR_1 = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .build();
+    private static final FlowRule DUAL_LINK_3_FLOW_1 = DefaultFlowEntry.builder().forDevice(DUAL_LINK_3)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(DUAL_LINK_3_FLOW_SELECTOR_1)
+            .withTreatment(DUAL_LINK_1_OUTPUT_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    static final FlowEntry DUAL_LINK_3_FLOW_ENTRY = new DefaultFlowEntry(DUAL_LINK_3_FLOW_1);
+
+    private static final TrafficSelector DUAL_LINK_3_FLOW_SELECTOR_2 = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(2))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .build();
+    private static final FlowRule DUAL_LINK_3_FLOW_2 = DefaultFlowEntry.builder().forDevice(DUAL_LINK_3)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(DUAL_LINK_3_FLOW_SELECTOR_2)
+            .withTreatment(DUAL_LINK_1_OUTPUT_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+
+    static final FlowEntry DUAL_LINK_3_FLOW_ENTRY_2 = new DefaultFlowEntry(DUAL_LINK_3_FLOW_2);
+
+    //Group with two buckets to output on port 2 and 3 of device 1 and 2
+
+    private static final GroupBucket BUCKET_2_DUAL =
+            DefaultGroupBucket.createSelectGroupBucket(DUAL_LINK_1_OUTPUT_TREATMENT);
+
+    private static final GroupBuckets BUCKETS_DUAL = new GroupBuckets(ImmutableList.of(BUCKET, BUCKET_2_DUAL));
+
+    static final Group DUAL_LINK_GROUP = new DefaultGroup(GROUP_ID, DUAL_LINK_1, Group.Type.SELECT, BUCKETS_DUAL);
+
+    //Clear Deferred
+    static final DeviceId DEFERRED_1 = DeviceId.deviceId("Deferred");
+
+    static final ConnectPoint DEFERRED_CP_1_IN = ConnectPoint.deviceConnectPoint(DEFERRED_1 + "/" + 1);
+    static final ConnectPoint DEFERRED_CP_2_OUT = ConnectPoint.deviceConnectPoint(DEFERRED_1 + "/" + 2);
+
+    //match on port 1 and apply deferred actions
+    private static final TrafficTreatment DEFERRED_1_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .transition(10)
+            .deferred()
+            .pushMpls()
+            .setMpls(MplsLabel.mplsLabel(100))
+            .build();
+    private static final FlowRule DEFERRED_FLOW = DefaultFlowEntry.builder().forDevice(DEFERRED_1)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(SINGLE_FLOW_SELECTOR)
+            .withTreatment(DEFERRED_1_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry DEFERRED_FLOW_ENTRY = new DefaultFlowEntry(DEFERRED_FLOW);
+
+    //Multicast Flow and Group Test
+    static final DeviceId MULTICAST_GROUP_FLOW_DEVICE = DeviceId.deviceId("MulticastGroupFlowDevice");
+
+    private static final TrafficSelector MULTICAST_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPDst(IpPrefix.valueOf("224.0.0.1/32"))
+            .matchEthDst(MacAddress.valueOf("01:00:5e:00:00:01"))
+            .build();
+
+    private static final FlowRule MULTICAST_GROUP_FLOW =
+            DefaultFlowEntry.builder().forDevice(MULTICAST_GROUP_FLOW_DEVICE)
+                    .forTable(0)
+                    .withPriority(100)
+                    .withSelector(MULTICAST_FLOW_SELECTOR)
+                    .withTreatment(GROUP_FLOW_TREATMENT)
+                    .fromApp(new DefaultApplicationId(0, "TestApp"))
+                    .makePermanent()
+                    .build();
+
+    static final FlowEntry MULTICAST_GROUP_FLOW_ENTRY = new DefaultFlowEntry(MULTICAST_GROUP_FLOW);
+
+    static final Group MULTICAST_GROUP = new DefaultGroup(GROUP_ID, MULTICAST_GROUP_FLOW_DEVICE,
+            Group.Type.SELECT, BUCKETS_MULTIPLE);
+
+    static final ConnectPoint MULTICAST_IN_CP = ConnectPoint.deviceConnectPoint(MULTICAST_GROUP_FLOW_DEVICE + "/" + 1);
+
+    static final ConnectPoint MULTICAST_OUT_CP = ConnectPoint.deviceConnectPoint(MULTICAST_GROUP_FLOW_DEVICE + "/" + 3);
+
+    static final ConnectPoint MULTICAST_OUT_CP_2 =
+            ConnectPoint.deviceConnectPoint(MULTICAST_GROUP_FLOW_DEVICE + "/" + 2);
+
+    //match on port 1, clear deferred actions and output
+    private static final TrafficTreatment DEFERRED_CLEAR_1_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .wipeDeferred()
+            .setOutput(PortNumber.portNumber(2))
+            .build();
+    private static final FlowRule DEFERRED_CLEAR_FLOW = DefaultFlowEntry.builder().forDevice(DEFERRED_1)
+            .forTable(10)
+            .withPriority(100)
+            .withSelector(SINGLE_FLOW_SELECTOR)
+            .withTreatment(DEFERRED_CLEAR_1_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry DEFERRED_CLEAR_FLOW_ENTRY = new DefaultFlowEntry(DEFERRED_CLEAR_FLOW);
+
+    //LLDP
+
+    static final DeviceId LLDP_FLOW_DEVICE = DeviceId.deviceId("LldpDevice");
+
+    private static final TrafficSelector LLDP_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchEthType(EthType.EtherType.LLDP.ethType().toShort())
+            .build();
+
+    private static final TrafficTreatment LLDP_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setOutput(PortNumber.CONTROLLER).build();
+    private static final FlowRule LLDP_FLOW = DefaultFlowEntry.builder().forDevice(LLDP_FLOW_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(LLDP_FLOW_SELECTOR)
+            .withTreatment(LLDP_FLOW_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry LLDP_FLOW_ENTRY = new DefaultFlowEntry(LLDP_FLOW);
+
+    static final ConnectPoint LLDP_FLOW_CP = ConnectPoint.deviceConnectPoint(LLDP_FLOW_DEVICE + "/" + 1);
+
+    //No Buckets
+
+    static final DeviceId NO_BUCKET_DEVICE = DeviceId.deviceId("nobucket");
+
+    private static final TrafficSelector NO_BUCKET_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .build();
+
+    private static final GroupId NO_BUCKET_GROUP_ID = GroupId.valueOf(1);
+
+    private static final TrafficTreatment NO_BUCKET_TREATMENT = DefaultTrafficTreatment.builder()
+            .group(NO_BUCKET_GROUP_ID)
+            .build();
+    private static final FlowRule NO_BUCKET_FLOW = DefaultFlowEntry.builder().forDevice(NO_BUCKET_DEVICE)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(NO_BUCKET_SELECTOR)
+            .withTreatment(NO_BUCKET_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry NO_BUCKET_ENTRY = new DefaultFlowEntry(NO_BUCKET_FLOW);
+
+    private static final GroupBuckets NO_BUCKETS = new GroupBuckets(ImmutableList.of());
+
+    static final Group NO_BUCKET_GROUP =
+            new DefaultGroup(NO_BUCKET_GROUP_ID, NO_BUCKET_DEVICE, Group.Type.SELECT, NO_BUCKETS);
+
+    static final ConnectPoint NO_BUCKET_CP = ConnectPoint.deviceConnectPoint(NO_BUCKET_DEVICE + "/" + 1);
+
+    //Dual Homing
+
+    static final DeviceId DUAL_HOME_DEVICE_1 = DeviceId.deviceId("DualHomeDevice1");
+
+    static final DeviceId DUAL_HOME_DEVICE_2 = DeviceId.deviceId("DualHomeDevice2");
+
+    static final DeviceId DUAL_HOME_DEVICE_3 = DeviceId.deviceId("DualHomeDevice3");
+
+    static final ConnectPoint DUAL_HOME_CP_1_1 = ConnectPoint.deviceConnectPoint(DUAL_HOME_DEVICE_1 + "/" + 1);
+    static final ConnectPoint DUAL_HOME_CP_1_2 = ConnectPoint.deviceConnectPoint(DUAL_HOME_DEVICE_1 + "/" + 2);
+    static final ConnectPoint DUAL_HOME_CP_1_3 = ConnectPoint.deviceConnectPoint(DUAL_HOME_DEVICE_1 + "/" + 3);
+
+    static final ConnectPoint DUAL_HOME_CP_2_1 = ConnectPoint.deviceConnectPoint(DUAL_HOME_DEVICE_2 + "/" + 1);
+    static final ConnectPoint DUAL_HOME_CP_2_2 = ConnectPoint.deviceConnectPoint(DUAL_HOME_DEVICE_2 + "/" + 2);
+
+    static final ConnectPoint DUAL_HOME_CP_3_1 = ConnectPoint.deviceConnectPoint(DUAL_HOME_DEVICE_3 + "/" + 1);
+    static final ConnectPoint DUAL_HOME_CP_3_2 = ConnectPoint.deviceConnectPoint(DUAL_HOME_DEVICE_3 + "/" + 2);
+
+
+    private static final TrafficSelector  DUAL_HOME_INPUT_FLOW_SELECTOR = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .build();
+
+    private static final GroupId DUAL_HOME_GROUP_ID = GroupId.valueOf(1);
+
+    private static final TrafficTreatment DUAL_HOME_GROUP_TREATMENT = DefaultTrafficTreatment.builder()
+            .group(DUAL_HOME_GROUP_ID)
+            .build();
+    private static final FlowRule DUAL_HOME_INPUT_FLOW = DefaultFlowEntry.builder().forDevice(DUAL_HOME_DEVICE_1)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(DUAL_HOME_INPUT_FLOW_SELECTOR)
+            .withTreatment(DUAL_HOME_GROUP_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry DUAL_HOME_FLOW_ENTRY = new DefaultFlowEntry(DUAL_HOME_INPUT_FLOW);
+
+    private static final TrafficTreatment DUAL_HOME_OUTPUT_1_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setOutput(PortNumber.portNumber(2)).build();
+    private static final TrafficTreatment DUAL_HOME_OUTPUT_2_FLOW_TREATMENT = DefaultTrafficTreatment.builder()
+            .setOutput(PortNumber.portNumber(3)).build();
+
+    private static final GroupBucket BUCKET_1_DUAL_HOMED =
+            DefaultGroupBucket.createSelectGroupBucket(DUAL_HOME_OUTPUT_1_FLOW_TREATMENT);
+
+    private static final GroupBucket BUCKET_2_DUAL_HOMED =
+            DefaultGroupBucket.createSelectGroupBucket(DUAL_HOME_OUTPUT_2_FLOW_TREATMENT);
+
+    private static final GroupBuckets BUCKETS_MULTIPLE_DUAL = new GroupBuckets(ImmutableList.of(BUCKET_1_DUAL_HOMED,
+            BUCKET_2_DUAL_HOMED));
+
+    static final Group DUAL_HOME_GROUP = new DefaultGroup(DUAL_HOME_GROUP_ID, DUAL_HOME_DEVICE_1,
+            Group.Type.SELECT, BUCKETS_MULTIPLE_DUAL);
+
+    private static final TrafficTreatment DUAL_HOME_TREATMENT = DefaultTrafficTreatment.builder()
+            .setOutput(PortNumber.portNumber("2"))
+            .build();
+    private static final FlowRule DUAL_HOME_OUT_FLOW = DefaultFlowEntry.builder().forDevice(DUAL_HOME_DEVICE_2)
+            .forTable(0)
+            .withPriority(100)
+            .withSelector(DUAL_HOME_INPUT_FLOW_SELECTOR)
+            .withTreatment(DUAL_HOME_TREATMENT)
+            .fromApp(new DefaultApplicationId(0, "TestApp"))
+            .makePermanent()
+            .build();
+    static final FlowEntry DUAL_HOME_OUT_FLOW_ENTRY = new DefaultFlowEntry(DUAL_HOME_OUT_FLOW);
+
+    //helper elements
+
+    static final String MASTER_1 = "Master1";
+
+    static final Host H1 = new DefaultHost(ProviderId.NONE, HostId.hostId(HOST_ONE), MacAddress.valueOf(100),
+            VlanId.NONE, new HostLocation(SINGLE_FLOW_DEVICE, PortNumber.portNumber(2), 0),
+            ImmutableSet.of(IpAddress.valueOf("127.0.0.2")));
+
+    static final Host H2 = new DefaultHost(ProviderId.NONE, HostId.hostId(HOST_TWO), MacAddress.valueOf(100),
+            VlanId.NONE, new HostLocation(TOPO_FLOW_3_DEVICE, PortNumber.portNumber(2), 0),
+            ImmutableSet.of(IpAddress.valueOf("127.0.0.3")));
+
+    static final Host DUAL_HOME_H = new DefaultHost(ProviderId.NONE, HostId.hostId(HOST_DUAL_HOMED),
+            MacAddress.valueOf(HOST_DUAL_HOMED_MAC),
+            VlanId.NONE, ImmutableSet.of(new HostLocation(DUAL_HOME_DEVICE_2, PortNumber.portNumber(2), 0),
+            new HostLocation(DUAL_HOME_DEVICE_3, PortNumber.portNumber(2), 0)),
+            ImmutableSet.of(IpAddress.valueOf("127.0.0.4")), true, DefaultAnnotations.builder().build());
+
+    static final TrafficSelector PACKET_OK = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchEthType(EthType.EtherType.IPV4.ethType().toShort())
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.2/32"))
+            .matchVlanId(VlanId.NONE)
+            .build();
+
+    static final TrafficSelector PACKET_OK_TOPO = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.3/32"))
+            .build();
+
+    static final TrafficSelector PACKET_ARP = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPDst(IpPrefix.valueOf("255.255.255.255/32"))
+            .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+            .build();
+
+    static final TrafficSelector PACKET_LLDP = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchEthType(EthType.EtherType.LLDP.ethType().toShort())
+            .build();
+
+    static final TrafficSelector PACKET_OK_MULTICAST = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchEthType(EthType.EtherType.IPV4.ethType().toShort())
+            .matchEthDst(MacAddress.valueOf("01:00:5e:00:00:01"))
+            .matchIPDst(IpPrefix.valueOf("224.0.0.1/32"))
+            .build();
+
+    static final TrafficSelector PACKET_DUAL_HOME = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchEthType(EthType.EtherType.IPV4.ethType().toShort())
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.4/32"))
+            .build();
+
+    static final TrafficSelector PACKET_FAIL = DefaultTrafficSelector.builder()
+            .matchInPort(PortNumber.portNumber(1))
+            .matchIPSrc(IpPrefix.valueOf("127.0.0.1/32"))
+            .matchIPDst(IpPrefix.valueOf("127.0.0.99/32"))
+            .build();
+}
diff --git a/apps/t3/app/src/test/java/org/onosproject/t3/impl/TroubleshootManagerTest.java b/apps/t3/app/src/test/java/org/onosproject/t3/impl/TroubleshootManagerTest.java
new file mode 100644
index 0000000..8ac76c8
--- /dev/null
+++ b/apps/t3/app/src/test/java/org/onosproject/t3/impl/TroubleshootManagerTest.java
@@ -0,0 +1,669 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.t3.impl;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.junit.Before;
+import org.junit.Test;
+import org.onlab.packet.ChassisId;
+import org.onlab.packet.EthType;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.VlanId;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.mastership.MastershipServiceAdapter;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DefaultAnnotations;
+import org.onosproject.net.DefaultDevice;
+import org.onosproject.net.DefaultLink;
+import org.onosproject.net.DefaultPort;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Host;
+import org.onosproject.net.Link;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceServiceAdapter;
+import org.onosproject.net.driver.DefaultDriver;
+import org.onosproject.net.driver.Driver;
+import org.onosproject.net.driver.DriverServiceAdapter;
+import org.onosproject.net.edge.EdgePortServiceAdapter;
+import org.onosproject.net.flow.FlowEntry;
+import org.onosproject.net.flow.FlowRuleServiceAdapter;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.EthTypeCriterion;
+import org.onosproject.net.flow.criteria.VlanIdCriterion;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupServiceAdapter;
+import org.onosproject.net.host.HostServiceAdapter;
+import org.onosproject.net.link.LinkServiceAdapter;
+import org.onosproject.net.provider.ProviderId;
+import org.onosproject.routeservice.ResolvedRoute;
+import org.onosproject.routeservice.RouteServiceAdapter;
+import org.onosproject.t3.api.StaticPacketTrace;
+import org.slf4j.Logger;
+
+import java.util.HashMap;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.onosproject.net.Device.Type.SWITCH;
+import static org.onosproject.t3.impl.T3TestObjects.*;
+import static org.onosproject.t3.impl.TroubleshootManager.PACKET_TO_CONTROLLER;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Test Class for Troubleshoot Manager.
+ */
+public class TroubleshootManagerTest {
+
+    private static final Logger log = getLogger(TroubleshootManager.class);
+
+    private TroubleshootManager mngr;
+
+    @Before
+    public void setUp() throws Exception {
+        mngr = new TroubleshootManager();
+        mngr.flowRuleService = new TestFlowRuleService();
+        mngr.hostService = new TestHostService();
+        mngr.linkService = new TestLinkService();
+        mngr.driverService = new TestDriverService();
+        mngr.groupService = new TestGroupService();
+        mngr.deviceService = new TestDeviceService();
+        mngr.mastershipService = new TestMastershipService();
+        mngr.edgePortService = new TestEdgePortService();
+        mngr.routeService = new TestRouteService();
+
+        assertNotNull("Manager should not be null", mngr);
+
+        assertNotNull("Flow rule Service should not be null", mngr.flowRuleService);
+        assertNotNull("Host Service should not be null", mngr.hostService);
+        assertNotNull("Group Service should not be null", mngr.groupService);
+        assertNotNull("Driver Service should not be null", mngr.driverService);
+        assertNotNull("Link Service should not be null", mngr.linkService);
+        assertNotNull("Device Service should not be null", mngr.deviceService);
+    }
+
+    /**
+     * Tests failure on non existent device.
+     */
+    @Test(expected = NullPointerException.class)
+    public void nonExistentDevice() {
+        StaticPacketTrace traceFail = mngr.trace(PACKET_OK, ConnectPoint.deviceConnectPoint("nonexistent" + "/1"));
+    }
+
+    /**
+     * Tests failure on offline device.
+     */
+    @Test
+    public void offlineDevice() {
+        StaticPacketTrace traceFail = mngr.trace(PACKET_OK, ConnectPoint.deviceConnectPoint(OFFLINE_DEVICE + "/1"));
+        assertNotNull("Trace should not be null", traceFail);
+        assertNull("Trace should have 0 output", traceFail.getGroupOuputs(SINGLE_FLOW_DEVICE));
+    }
+
+    /**
+     * Tests failure on same output.
+     */
+    @Test
+    public void sameOutput() {
+        StaticPacketTrace traceFail = mngr.trace(PACKET_OK, SAME_OUTPUT_FLOW_CP);
+        assertNotNull("Trace should not be null", traceFail);
+        assertTrue("Trace should be unsuccessful",
+                traceFail.resultMessage().contains("is same as initial input"));
+        log.info("trace {}", traceFail.resultMessage());
+    }
+
+    /**
+     * Tests ARP to controller.
+     */
+    @Test
+    public void arpToController() {
+        StaticPacketTrace traceSuccess = mngr.trace(PACKET_ARP, ARP_FLOW_CP);
+        assertNotNull("Trace should not be null", traceSuccess);
+        assertTrue("Trace should be successful",
+                traceSuccess.resultMessage().contains(PACKET_TO_CONTROLLER));
+        assertTrue("Master should be Master1",
+                traceSuccess.resultMessage().contains(MASTER_1));
+        ConnectPoint connectPoint = traceSuccess.getGroupOuputs(ARP_FLOW_DEVICE).get(0).getOutput();
+        assertEquals("Packet Should go to CONTROLLER", PortNumber.CONTROLLER, connectPoint.port());
+        assertNull("VlanId should be null", traceSuccess.getGroupOuputs(ARP_FLOW_DEVICE).get(0)
+                .getFinalPacket().getCriterion(Criterion.Type.VLAN_VID));
+        log.info("trace {}", traceSuccess.resultMessage());
+    }
+
+
+    /**
+     * Tests failure on device with no flows.
+     */
+    @Test
+    public void noFlows() {
+        StaticPacketTrace traceFail = mngr.trace(PACKET_OK, ConnectPoint.deviceConnectPoint("test/1"));
+        assertNotNull("Trace should not be null", traceFail);
+        assertNull("Trace should have 0 output", traceFail.getGroupOuputs(SINGLE_FLOW_DEVICE));
+        log.info("trace {}", traceFail.resultMessage());
+    }
+
+    /**
+     * Test group with no buckets.
+     */
+    @Test
+    public void noBucketsTest() throws Exception {
+
+        StaticPacketTrace traceFail = mngr.trace(PACKET_OK, NO_BUCKET_CP);
+        assertNotNull("Trace should not be null", traceFail);
+        assertTrue("Trace should be unsuccessful",
+                traceFail.resultMessage().contains("no buckets"));
+        log.info("trace {}", traceFail.resultMessage());
+
+    }
+
+    /**
+     * Test a single flow rule that has output port in it.
+     */
+    @Test
+    public void testSingleFlowRule() {
+
+        testSuccess(PACKET_OK, SINGLE_FLOW_IN_CP, SINGLE_FLOW_DEVICE, SINGLE_FLOW_OUT_CP, 1, 1);
+
+        testFailure(PACKET_FAIL, SINGLE_FLOW_IN_CP, SINGLE_FLOW_DEVICE);
+    }
+
+    /**
+     * Tests two flow rule the last one of which has output port in it.
+     */
+    @Test
+    public void testDualFlowRule() {
+
+        //Test Success
+
+        StaticPacketTrace traceSuccess = testSuccess(PACKET_OK, DUAL_FLOW_IN_CP, DUAL_FLOW_DEVICE,
+                DUAL_FLOW_OUT_CP, 1, 1);
+
+        //Testing Vlan
+        Criterion criterion = traceSuccess.getGroupOuputs(DUAL_FLOW_DEVICE).get(0).
+                getFinalPacket().getCriterion(Criterion.Type.VLAN_VID);
+        assertNotNull("Packet Should have Vlan", criterion);
+
+        VlanIdCriterion vlanIdCriterion = (VlanIdCriterion) criterion;
+
+        assertEquals("Vlan should be 100", VlanId.vlanId((short) 100), vlanIdCriterion.vlanId());
+
+        //Test Faliure
+        testFailure(PACKET_FAIL, DUAL_FLOW_IN_CP, DUAL_FLOW_DEVICE);
+
+    }
+
+    /**
+     * Test a single flow rule that points to a group with output port in it.
+     */
+    @Test
+    public void flowAndGroup() throws Exception {
+
+        StaticPacketTrace traceSuccess = testSuccess(PACKET_OK, GROUP_FLOW_IN_CP, GROUP_FLOW_DEVICE,
+                GROUP_FLOW_OUT_CP, 1, 1);
+
+        assertTrue("Wrong Output Group", traceSuccess.getGroupOuputs(GROUP_FLOW_DEVICE)
+                .get(0).getGroups().contains(GROUP));
+        assertEquals("Packet should not have MPLS Label", EthType.EtherType.IPV4.ethType(),
+                ((EthTypeCriterion) traceSuccess.getGroupOuputs(GROUP_FLOW_DEVICE)
+                        .get(0).getFinalPacket().getCriterion(Criterion.Type.ETH_TYPE)).ethType());
+        assertNull("Packet should not have MPLS Label", traceSuccess.getGroupOuputs(GROUP_FLOW_DEVICE)
+                .get(0).getFinalPacket().getCriterion(Criterion.Type.MPLS_LABEL));
+        assertNull("Packet should not have MPLS Label", traceSuccess.getGroupOuputs(GROUP_FLOW_DEVICE)
+                .get(0).getFinalPacket().getCriterion(Criterion.Type.MPLS_BOS));
+
+    }
+
+    /**
+     * Test path through a 3 device topology.
+     */
+    @Test
+    public void singlePathTopology() throws Exception {
+
+        StaticPacketTrace traceSuccess = testSuccess(PACKET_OK_TOPO, TOPO_FLOW_1_IN_CP,
+                TOPO_FLOW_3_DEVICE, TOPO_FLOW_3_OUT_CP, 1, 1);
+
+        assertTrue("Incorrect path",
+                traceSuccess.getCompletePaths().get(0).contains(TOPO_FLOW_2_IN_CP));
+        assertTrue("Incorrect path",
+                traceSuccess.getCompletePaths().get(0).contains(TOPO_FLOW_2_OUT_CP));
+        assertTrue("Incorrect path",
+                traceSuccess.getCompletePaths().get(0).contains(TOPO_FLOW_3_IN_CP));
+
+    }
+
+    /**
+     * Test path through a 4 device topology with first device that has groups with multiple output buckets.
+     */
+    @Test
+    public void testGroupTopo() throws Exception {
+
+        StaticPacketTrace traceSuccess = testSuccess(PACKET_OK_TOPO, TOPO_FLOW_IN_CP,
+                TOPO_FLOW_3_DEVICE, TOPO_FLOW_3_OUT_CP, 2, 1);
+
+        log.info("{}", traceSuccess);
+
+        assertTrue("Incorrect groups",
+                traceSuccess.getGroupOuputs(TOPO_GROUP_FLOW_DEVICE).get(0).getGroups().contains(TOPO_GROUP));
+        assertTrue("Incorrect bucket",
+                traceSuccess.getGroupOuputs(TOPO_GROUP_FLOW_DEVICE).get(1).getGroups().contains(TOPO_GROUP));
+    }
+
+    /**
+     * Test HW support in a single device with 2 flow rules to check hit of static HW rules.
+     */
+    @Test
+    public void hardwareTest() throws Exception {
+
+        StaticPacketTrace traceSuccess = testSuccess(PACKET_OK, HARDWARE_DEVICE_IN_CP,
+                HARDWARE_DEVICE, HARDWARE_DEVICE_OUT_CP, 1, 1);
+
+        assertEquals("wrong ETH type", EthType.EtherType.IPV4.ethType(),
+                ((EthTypeCriterion) traceSuccess.getGroupOuputs(HARDWARE_DEVICE).get(0).getFinalPacket()
+                        .getCriterion(Criterion.Type.ETH_TYPE)).ethType());
+
+    }
+
+    /**
+     * Test that HW has two rules on table 10 for untagged packets.
+     */
+    @Test
+    public void hardwareTable10Test() throws Exception {
+
+        StaticPacketTrace traceSuccess = testSuccess(PACKET_OK, HARDWARE_DEVICE_10_IN_CP,
+                HARDWARE_DEVICE_10, HARDWARE_DEVICE_10_OUT_CP, 1, 1);
+
+        assertTrue("Second flow rule is absent", traceSuccess.getFlowsForDevice(HARDWARE_DEVICE_10)
+                .contains(HARDWARE_10_SECOND_FLOW_ENTRY));
+
+    }
+
+    /**
+     * Test dual links between 3 topology elements.
+     */
+    @Test
+    public void dualLinks() throws Exception {
+
+        StaticPacketTrace traceSuccess = testSuccess(PACKET_OK, DUAL_LINK_1_CP_1_IN,
+                DUAL_LINK_3, DUAL_LINK_3_CP_3_OUT, 4, 1);
+
+        //TODO tests
+
+    }
+
+    /**
+     * Test proper clear deferred behaviour.
+     */
+    @Test
+    public void clearDeferred() throws Exception {
+
+        StaticPacketTrace traceSuccess = testSuccess(PACKET_OK, DEFERRED_CP_1_IN,
+                DEFERRED_1, DEFERRED_CP_2_OUT, 1, 1);
+
+        assertNull("MPLS should have been not applied due to clear deferred", traceSuccess
+                .getGroupOuputs(DEFERRED_1).get(0).getFinalPacket().getCriterion(Criterion.Type.MPLS_LABEL));
+
+    }
+
+
+    /**
+     * Test LLDP output to controller.
+     */
+    @Test
+    public void lldpToController() {
+        StaticPacketTrace traceSuccess = mngr.trace(PACKET_LLDP, LLDP_FLOW_CP);
+        assertNotNull("Trace should not be null", traceSuccess);
+        assertTrue("Trace should be successful",
+                traceSuccess.resultMessage().contains("Packet goes to the controller"));
+        assertTrue("Master should be Master1",
+                traceSuccess.resultMessage().contains(MASTER_1));
+        ConnectPoint connectPoint = traceSuccess.getGroupOuputs(LLDP_FLOW_DEVICE).get(0).getOutput();
+        assertEquals("Packet Should go to CONTROLLER", PortNumber.CONTROLLER, connectPoint.port());
+        log.info("trace {}", traceSuccess.resultMessage());
+    }
+
+    /**
+     * Test multicast in single device.
+     */
+    @Test
+    public void multicastTest() throws Exception {
+
+        StaticPacketTrace traceSuccess = mngr.trace(PACKET_OK_MULTICAST, MULTICAST_IN_CP);
+
+        log.info("trace {}", traceSuccess);
+
+        log.info("trace {}", traceSuccess.resultMessage());
+
+        assertNotNull("trace should not be null", traceSuccess);
+        assertEquals("Trace should have " + 2 + " output", 2,
+                traceSuccess.getGroupOuputs(MULTICAST_GROUP_FLOW_DEVICE).size());
+        assertEquals("Trace should only have " + 2 + "output", 2,
+                traceSuccess.getCompletePaths().size());
+        assertTrue("Trace should be successful",
+                traceSuccess.resultMessage().contains("reached output"));
+        assertEquals("Incorrect Output CP", MULTICAST_OUT_CP_2,
+                traceSuccess.getGroupOuputs(MULTICAST_GROUP_FLOW_DEVICE).get(0).getOutput());
+        assertEquals("Incorrect Output CP", MULTICAST_OUT_CP,
+                traceSuccess.getGroupOuputs(MULTICAST_GROUP_FLOW_DEVICE).get(1).getOutput());
+
+    }
+
+    /**
+     * Tests dual homing of a host.
+     */
+    @Test
+    public void dualhomedTest() throws Exception {
+        StaticPacketTrace traceSuccess = mngr.trace(PACKET_DUAL_HOME, DUAL_HOME_CP_1_1);
+
+        assertNotNull("trace should not be null", traceSuccess);
+        assertTrue("Should have 2 output paths", traceSuccess.getCompletePaths().size() == 2);
+        assertTrue("Should contain proper path", traceSuccess.getCompletePaths()
+                .contains(ImmutableList.of(DUAL_HOME_CP_1_1, DUAL_HOME_CP_1_2, DUAL_HOME_CP_2_1, DUAL_HOME_CP_2_2)));
+        assertTrue("Should contain proper path", traceSuccess.getCompletePaths()
+                .contains(ImmutableList.of(DUAL_HOME_CP_1_1, DUAL_HOME_CP_1_3, DUAL_HOME_CP_3_1, DUAL_HOME_CP_3_2)));
+
+    }
+
+
+    private StaticPacketTrace testSuccess(TrafficSelector packet, ConnectPoint in, DeviceId deviceId, ConnectPoint out,
+                                          int paths, int outputs) {
+        StaticPacketTrace traceSuccess = mngr.trace(packet, in);
+
+        log.info("trace {}", traceSuccess);
+
+        log.info("trace {}", traceSuccess.resultMessage());
+
+        assertNotNull("trace should not be null", traceSuccess);
+        assertEquals("Trace should have " + outputs + " output", outputs,
+                traceSuccess.getGroupOuputs(deviceId).size());
+        assertEquals("Trace should only have " + paths + "output", paths, traceSuccess.getCompletePaths().size());
+        assertTrue("Trace should be successful",
+                traceSuccess.resultMessage().contains("Reached required destination Host"));
+        assertEquals("Incorrect Output CP", out,
+                traceSuccess.getGroupOuputs(deviceId).get(0).getOutput());
+
+        return traceSuccess;
+    }
+
+    private void testFailure(TrafficSelector packet, ConnectPoint in, DeviceId deviceId) {
+        StaticPacketTrace traceFail = mngr.trace(packet, in);
+
+        log.info("trace {}", traceFail.resultMessage());
+
+        assertNotNull("Trace should not be null", traceFail);
+        assertNull("Trace should have 0 output", traceFail.getGroupOuputs(deviceId));
+    }
+
+    private class TestFlowRuleService extends FlowRuleServiceAdapter {
+        @Override
+        public Iterable<FlowEntry> getFlowEntriesByState(DeviceId deviceId, FlowEntry.FlowEntryState state) {
+            if (deviceId.equals(SINGLE_FLOW_DEVICE)) {
+                return ImmutableList.of(SINGLE_FLOW_ENTRY);
+            } else if (deviceId.equals(DUAL_FLOW_DEVICE)) {
+                return ImmutableList.of(FIRST_FLOW_ENTRY, SECOND_FLOW_ENTRY);
+            } else if (deviceId.equals(GROUP_FLOW_DEVICE)) {
+                return ImmutableList.of(GROUP_FLOW_ENTRY);
+            } else if (deviceId.equals(TOPO_FLOW_DEVICE) ||
+                    deviceId.equals(TOPO_FLOW_2_DEVICE) ||
+                    deviceId.equals(TOPO_FLOW_3_DEVICE) ||
+                    deviceId.equals(TOPO_FLOW_4_DEVICE)) {
+                return ImmutableList.of(TOPO_SINGLE_FLOW_ENTRY, TOPO_SECOND_INPUT_FLOW_ENTRY);
+            } else if (deviceId.equals(TOPO_GROUP_FLOW_DEVICE)) {
+                return ImmutableList.of(TOPO_GROUP_FLOW_ENTRY);
+            } else if (deviceId.equals(HARDWARE_DEVICE)) {
+                return ImmutableList.of(HARDWARE_ETH_FLOW_ENTRY, HARDWARE_FLOW_ENTRY);
+            } else if (deviceId.equals(SAME_OUTPUT_FLOW_DEVICE)) {
+                return ImmutableList.of(SAME_OUTPUT_FLOW_ENTRY);
+            } else if (deviceId.equals(ARP_FLOW_DEVICE)) {
+                return ImmutableList.of(ARP_FLOW_ENTRY);
+            } else if (deviceId.equals(DUAL_LINK_1)) {
+                return ImmutableList.of(DUAL_LINK_1_GROUP_FLOW_ENTRY);
+            } else if (deviceId.equals(DUAL_LINK_2)) {
+                return ImmutableList.of(DUAL_LINK_1_GROUP_FLOW_ENTRY, DUAL_LINK_2_GROUP_FLOW_ENTRY);
+            } else if (deviceId.equals(DUAL_LINK_3)) {
+                return ImmutableList.of(DUAL_LINK_3_FLOW_ENTRY, DUAL_LINK_3_FLOW_ENTRY_2);
+            } else if (deviceId.equals(DEFERRED_1)) {
+                return ImmutableList.of(DEFERRED_FLOW_ENTRY, DEFERRED_CLEAR_FLOW_ENTRY);
+            } else if (deviceId.equals(HARDWARE_DEVICE_10)) {
+                return ImmutableList.of(HARDWARE_10_FLOW_ENTRY, HARDWARE_10_SECOND_FLOW_ENTRY,
+                        HARDWARE_10_OUTPUT_FLOW_ENTRY);
+            } else if (deviceId.equals(LLDP_FLOW_DEVICE)) {
+                return ImmutableList.of(LLDP_FLOW_ENTRY);
+            } else if (deviceId.equals(MULTICAST_GROUP_FLOW_DEVICE)) {
+                return ImmutableList.of(MULTICAST_GROUP_FLOW_ENTRY);
+            } else if (deviceId.equals(NO_BUCKET_DEVICE)) {
+                return ImmutableList.of(NO_BUCKET_ENTRY);
+            } else if (deviceId.equals(DUAL_HOME_DEVICE_1)) {
+                return ImmutableList.of(DUAL_HOME_FLOW_ENTRY);
+            } else if (deviceId.equals(DUAL_HOME_DEVICE_2) || deviceId.equals(DUAL_HOME_DEVICE_3)) {
+                return ImmutableList.of(DUAL_HOME_OUT_FLOW_ENTRY);
+            }
+            return ImmutableList.of();
+        }
+    }
+
+    private class TestDriverService extends DriverServiceAdapter {
+        @Override
+        public Driver getDriver(DeviceId deviceId) {
+            if (deviceId.equals(HARDWARE_DEVICE) || deviceId.equals(HARDWARE_DEVICE_10)) {
+                return new DefaultDriver("ofdpa", ImmutableList.of(),
+                        "test", "test", "test", new HashMap<>(), new HashMap<>());
+            }
+            return new DefaultDriver("NotHWDriver", ImmutableList.of(),
+                    "test", "test", "test", new HashMap<>(), new HashMap<>());
+        }
+    }
+
+    private class TestGroupService extends GroupServiceAdapter {
+        @Override
+        public Iterable<Group> getGroups(DeviceId deviceId) {
+            if (deviceId.equals(GROUP_FLOW_DEVICE)) {
+                return ImmutableList.of(GROUP);
+            } else if (deviceId.equals(TOPO_GROUP_FLOW_DEVICE)) {
+                return ImmutableList.of(TOPO_GROUP);
+            } else if (deviceId.equals(DUAL_LINK_1) || deviceId.equals(DUAL_LINK_2)) {
+                return ImmutableList.of(DUAL_LINK_GROUP);
+            } else if (deviceId.equals(MULTICAST_GROUP_FLOW_DEVICE)) {
+                return ImmutableList.of(MULTICAST_GROUP);
+            } else if (deviceId.equals(NO_BUCKET_DEVICE)) {
+                return ImmutableList.of(NO_BUCKET_GROUP);
+            } else if (deviceId.equals(DUAL_HOME_DEVICE_1)) {
+                return ImmutableList.of(DUAL_HOME_GROUP);
+            }
+            return ImmutableList.of();
+        }
+    }
+
+    private class TestHostService extends HostServiceAdapter {
+        @Override
+        public Set<Host> getConnectedHosts(ConnectPoint connectPoint) {
+            if (connectPoint.equals(TOPO_FLOW_3_OUT_CP)) {
+                return ImmutableSet.of(H2);
+            } else if (connectPoint.equals(DUAL_LINK_1_CP_2_OUT) || connectPoint.equals(DUAL_LINK_1_CP_3_OUT) ||
+                    connectPoint.equals(DUAL_LINK_2_CP_2_OUT) || connectPoint.equals(DUAL_LINK_2_CP_3_OUT)) {
+                return ImmutableSet.of();
+            }
+            if (connectPoint.equals(SINGLE_FLOW_OUT_CP) ||
+                    connectPoint.equals(DUAL_FLOW_OUT_CP) ||
+                    connectPoint.equals(GROUP_FLOW_OUT_CP) ||
+                    connectPoint.equals(HARDWARE_DEVICE_OUT_CP) ||
+                    connectPoint.equals(HARDWARE_DEVICE_10_OUT_CP) ||
+                    connectPoint.equals(DEFERRED_CP_2_OUT) ||
+                    connectPoint.equals(DUAL_LINK_3_CP_3_OUT)) {
+                return ImmutableSet.of(H1);
+            }
+            if (connectPoint.equals(DUAL_HOME_CP_2_2) || connectPoint.equals(DUAL_HOME_CP_3_2)) {
+                return ImmutableSet.of(DUAL_HOME_H);
+            }
+            return ImmutableSet.of();
+        }
+
+        @Override
+        public Set<Host> getHostsByMac(MacAddress mac) {
+            if (mac.equals(H1.mac())) {
+                return ImmutableSet.of(H1);
+            } else if (mac.equals(H2.mac())) {
+                return ImmutableSet.of(H2);
+            } else if (mac.equals(DUAL_HOME_H.mac())) {
+                return ImmutableSet.of(DUAL_HOME_H);
+            }
+            return ImmutableSet.of();
+        }
+
+        @Override
+        public Set<Host> getHostsByIp(IpAddress ip) {
+            if ((H1.ipAddresses().contains(ip))) {
+                return ImmutableSet.of(H1);
+            } else if ((H2.ipAddresses().contains(ip))) {
+                return ImmutableSet.of(H2);
+            } else if ((DUAL_HOME_H.ipAddresses().contains(ip))) {
+                return ImmutableSet.of(DUAL_HOME_H);
+            }
+            return ImmutableSet.of();
+        }
+    }
+
+    private class TestLinkService extends LinkServiceAdapter {
+        @Override
+        public Set<Link> getEgressLinks(ConnectPoint connectPoint) {
+            if (connectPoint.equals(TOPO_FLOW_1_OUT_CP)
+                    || connectPoint.equals(TOPO_FLOW_OUT_CP_1)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(connectPoint)
+                        .dst(TOPO_FLOW_2_IN_CP)
+                        .build());
+            } else if (connectPoint.equals(TOPO_FLOW_2_OUT_CP)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(TOPO_FLOW_2_OUT_CP)
+                        .dst(TOPO_FLOW_3_IN_CP)
+                        .build());
+            } else if (connectPoint.equals(TOPO_FLOW_OUT_CP_2)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(TOPO_FLOW_OUT_CP_2)
+                        .dst(TOPO_FLOW_4_IN_CP)
+                        .build());
+            } else if (connectPoint.equals(TOPO_FLOW_4_OUT_CP)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(TOPO_FLOW_4_OUT_CP)
+                        .dst(TOPO_FLOW_3_IN_2_CP)
+                        .build());
+            } else if (connectPoint.equals(DUAL_LINK_1_CP_2_OUT)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(DUAL_LINK_1_CP_2_OUT)
+                        .dst(DUAL_LINK_2_CP_1_IN)
+                        .build());
+            } else if (connectPoint.equals(DUAL_LINK_1_CP_3_OUT)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(DUAL_LINK_1_CP_3_OUT)
+                        .dst(DUAL_LINK_2_CP_4_IN)
+                        .build());
+            } else if (connectPoint.equals(DUAL_LINK_2_CP_2_OUT)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(DUAL_LINK_2_CP_2_OUT)
+                        .dst(DUAL_LINK_3_CP_1_IN)
+                        .build());
+            } else if (connectPoint.equals(DUAL_LINK_2_CP_3_OUT)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(DUAL_LINK_2_CP_3_OUT)
+                        .dst(DUAL_LINK_3_CP_2_IN)
+                        .build());
+            } else if (connectPoint.equals(DUAL_HOME_CP_1_2)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(DUAL_HOME_CP_1_2)
+                        .dst(DUAL_HOME_CP_2_1)
+                        .build());
+            } else if (connectPoint.equals(DUAL_HOME_CP_1_3)) {
+                return ImmutableSet.of(DefaultLink.builder()
+                        .providerId(ProviderId.NONE)
+                        .type(Link.Type.DIRECT)
+                        .src(DUAL_HOME_CP_1_3)
+                        .dst(DUAL_HOME_CP_3_1)
+                        .build());
+            }
+            return ImmutableSet.of();
+        }
+    }
+
+    private class TestDeviceService extends DeviceServiceAdapter {
+        @Override
+        public Device getDevice(DeviceId deviceId) {
+            if (deviceId.equals(DeviceId.deviceId("nonexistent"))) {
+                return null;
+            }
+            return new DefaultDevice(ProviderId.NONE, DeviceId.deviceId("test"), SWITCH,
+                    "test", "test", "test", "test", new ChassisId(),
+                    DefaultAnnotations.builder().set("foo", "bar").build());
+        }
+
+        @Override
+        public Port getPort(ConnectPoint cp) {
+            return new DefaultPort(null, cp.port(), true, DefaultAnnotations.builder().build());
+        }
+
+        @Override
+        public boolean isAvailable(DeviceId deviceId) {
+            return !deviceId.equals(OFFLINE_DEVICE);
+        }
+    }
+
+    private class TestEdgePortService extends EdgePortServiceAdapter {
+
+        @Override
+        public boolean isEdgePoint(ConnectPoint point) {
+            return point.equals(MULTICAST_OUT_CP) ||
+                    point.equals(MULTICAST_OUT_CP_2);
+        }
+    }
+
+    private class TestRouteService extends RouteServiceAdapter {
+        @Override
+        public Optional<ResolvedRoute> longestPrefixLookup(IpAddress ip) {
+            return Optional.empty();
+        }
+    }
+
+    private class TestMastershipService extends MastershipServiceAdapter {
+        @Override
+        public NodeId getMasterFor(DeviceId deviceId) {
+            return NodeId.nodeId(MASTER_1);
+        }
+    }
+}
\ No newline at end of file
diff --git a/apps/t3/app/src/test/java/org/onosproject/t3/impl/TroubleshootUtilsTest.java b/apps/t3/app/src/test/java/org/onosproject/t3/impl/TroubleshootUtilsTest.java
new file mode 100644
index 0000000..c8195c0
--- /dev/null
+++ b/apps/t3/app/src/test/java/org/onosproject/t3/impl/TroubleshootUtilsTest.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.t3.impl;
+
+import org.junit.Test;
+import org.onlab.packet.MacAddress;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for util methods of the Trellis Troubleshoot Toolkit.
+ */
+public class TroubleshootUtilsTest {
+
+    @Test
+    public void testMacMatch() {
+
+        MacAddress min = MacAddress.valueOf("01:00:5E:00:00:00");
+        MacAddress mask = MacAddress.valueOf("FF:FF:FF:80:00:00");
+        MacAddress macOk = MacAddress.valueOf("01:00:5E:00:00:01");
+
+        assertTrue("False on correct match", TroubleshootUtils.compareMac(macOk, min, mask));
+
+        MacAddress macWrong = MacAddress.valueOf("01:00:5E:80:00:00");
+
+        assertFalse("True on false match", TroubleshootUtils.compareMac(macWrong, min, mask));
+
+        MacAddress maskEmpty = MacAddress.valueOf("00:00:00:00:00:00");
+
+        assertTrue("False on empty Mask", TroubleshootUtils.compareMac(macOk, min, maskEmpty));
+
+        MacAddress maskFull = MacAddress.valueOf("FF:FF:FF:FF:FF:FF");
+
+        assertFalse("True on full Mask", TroubleshootUtils.compareMac(macOk, min, maskFull));
+
+    }
+}
\ No newline at end of file