Initial support VM and container communication via POD and service IP

Change-Id: Ic87beee6ed122ec5551370c2b6a2789edf8fba5b
diff --git a/apps/k8s-networking/app/BUILD b/apps/k8s-networking/app/BUILD
index 725e27d..87c974e 100644
--- a/apps/k8s-networking/app/BUILD
+++ b/apps/k8s-networking/app/BUILD
@@ -5,6 +5,8 @@
     "//apps/k8s-node/api:onos-apps-k8s-node-api",
     "//apps/k8s-networking/api:onos-apps-k8s-networking-api",
     "@commons_net//jar",
+    "@jersey_client//jar",
+    "@json//jar",
     "@k8s_client//jar",
     "@k8s_model//jar",
     "@okhttp//jar",
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/DistributedK8sServiceStore.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/DistributedK8sServiceStore.java
index 225158a..07b122f 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/DistributedK8sServiceStore.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/DistributedK8sServiceStore.java
@@ -23,6 +23,7 @@
 import io.fabric8.kubernetes.api.model.LoadBalancerStatus;
 import io.fabric8.kubernetes.api.model.ManagedFieldsEntry;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
+import io.fabric8.kubernetes.api.model.OwnerReference;
 import io.fabric8.kubernetes.api.model.Service;
 import io.fabric8.kubernetes.api.model.ServicePort;
 import io.fabric8.kubernetes.api.model.ServiceSpec;
@@ -91,6 +92,7 @@
             .register(ClientIPConfig.class)
             .register(ManagedFieldsEntry.class)
             .register(FieldsV1.class)
+            .register(OwnerReference.class)
             .register(LinkedHashMap.class)
             .register(Collection.class)
             .build();
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
new file mode 100644
index 0000000..0c68e20
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
@@ -0,0 +1,203 @@
+/*
+ * Copyright 2020-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snetworking.api.K8sNetwork;
+import org.onosproject.k8snetworking.api.K8sNetworkService;
+import org.onosproject.k8snode.api.K8sNode;
+import org.onosproject.k8snode.api.K8sNodeEvent;
+import org.onosproject.k8snode.api.K8sNodeListener;
+import org.onosproject.k8snode.api.K8sNodeService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.impl.OsgiPropertyConstants.SERVICE_IP_CIDR_DEFAULT;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.getBclassIpPrefixFromCidr;
+import static org.onosproject.k8snode.api.K8sApiConfig.Mode.PASSTHROUGH;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Provides kubernetes and openstack integration feature.
+ */
+@Component(immediate = true)
+public class K8sOpenstackIntegrationHandler {
+
+    private final Logger log = getLogger(getClass());
+
+    private static final String K8S_NODE_IP = "k8sNodeIp";
+    private static final String OS_K8S_INT_PORT_NAME = "osK8sIntPortName";
+    private static final String POD_CIDR = "podCidr";
+    private static final String SERVICE_CIDR = "serviceCidr";
+    private static final String POD_GW_IP = "podGwIp";
+    private static final String K8S_INT_OS_PORT_MAC = "k8sIntOsPortMac";
+    private static final String ONOS_PORT = "8181";
+    private static final String OS_K8S_INTEGRATION_EP = "onos/openstacknetworking/integration/";
+    private static final String ONOS_USERNAME = "karaf";
+    private static final String ONOS_PASSWORD = "karaf";
+    private static final String B_CLASS_SUFFIX = ".0.0/16";
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected CoreService coreService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected ClusterService clusterService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected LeadershipService leadershipService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected K8sNodeService k8sNodeService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected K8sNetworkService k8sNetworkService;
+
+    private final InternalK8sNodeListener k8sNodeListener =
+            new InternalK8sNodeListener();
+    private final ExecutorService eventExecutor = newSingleThreadExecutor(
+            groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+
+    private ApplicationId appId;
+    private NodeId localNodeId;
+
+    @Activate
+    protected void activate() {
+        appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
+        localNodeId = clusterService.getLocalNode().id();
+        k8sNodeService.addListener(k8sNodeListener);
+
+
+        log.info("Started");
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        k8sNodeService.removeListener(k8sNodeListener);
+
+        log.info("Stopped");
+    }
+
+    private void setCniPtNodeRules(K8sNode k8sNode, boolean install) {
+        K8sNetwork network = k8sNetworkService.network(k8sNode.hostname());
+        String k8sNodeIp = k8sNode.nodeIp().toString();
+        String gatewayIp = network.gatewayIp().toString();
+        String nodePodCidr = network.cidr();
+        String srcPodPrefix = getBclassIpPrefixFromCidr(nodePodCidr);
+        String podCidr = srcPodPrefix + B_CLASS_SUFFIX;
+        String osK8sIntPortName = k8sNode.osToK8sIntgPatchPortName();
+        String k8sIntOsPortMac = k8sNode.portMacByName(k8sNode.k8sIntgToOsPatchPortName()).toString();
+
+        String path = install ? "node/pt-install" : "node/pt-uninstall";
+
+        String jsonString = "";
+
+        try {
+            jsonString = new JSONObject()
+                    .put(K8S_NODE_IP, k8sNodeIp)
+                    .put(POD_GW_IP, gatewayIp)
+                    .put(POD_CIDR, podCidr)
+                    .put(SERVICE_CIDR, SERVICE_IP_CIDR_DEFAULT)
+                    .put(OS_K8S_INT_PORT_NAME, osK8sIntPortName)
+                    .put(K8S_INT_OS_PORT_MAC, k8sIntOsPortMac)
+                    .toString();
+            log.info("push integration configuration {}", jsonString);
+        } catch (JSONException e) {
+            log.error("Failed to generate JSON string");
+            return;
+        }
+
+        HttpAuthenticationFeature feature =
+                HttpAuthenticationFeature.basic(ONOS_USERNAME, ONOS_PASSWORD);
+
+        final Client client = ClientBuilder.newClient();
+        client.register(feature);
+        String host = "http://" + k8sNode.managementIp().toString() + ":" + ONOS_PORT + "/";
+        String endpoint = host + OS_K8S_INTEGRATION_EP;
+        WebTarget wt = client.target(endpoint).path(path);
+        Response response = wt.request(MediaType.APPLICATION_JSON_TYPE)
+                .put(Entity.json(jsonString));
+        final int status = response.getStatus();
+
+        if (status != 200) {
+            log.error("Failed to install/uninstall openstack k8s CNI PT rules.");
+        }
+    }
+
+    private class InternalK8sNodeListener implements K8sNodeListener {
+
+        @Override
+        public boolean isRelevant(K8sNodeEvent event) {
+            return event.subject().mode() == PASSTHROUGH;
+        }
+
+        private boolean isRelevantHelper() {
+            return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+        }
+
+        @Override
+        public void event(K8sNodeEvent event) {
+            switch (event.type()) {
+                case K8S_NODE_COMPLETE:
+                    eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+                    break;
+                case K8S_NODE_INCOMPLETE:
+                    eventExecutor.execute(() -> processNodeIncompletion(event.subject()));
+                    break;
+                default:
+                    break;
+            }
+        }
+
+        private void processNodeCompletion(K8sNode k8sNode) {
+            if (!isRelevantHelper()) {
+                return;
+            }
+
+            setCniPtNodeRules(k8sNode, true);
+        }
+
+        private void processNodeIncompletion(K8sNode k8sNode) {
+            if (!isRelevantHelper()) {
+                return;
+            }
+
+            setCniPtNodeRules(k8sNode, false);
+        }
+    }
+}
\ No newline at end of file
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodPortMapper.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodPortMapper.java
index 5239ee4..dfc77cf 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodPortMapper.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodPortMapper.java
@@ -170,10 +170,13 @@
                     annotations.put(PORT_ID, p.portId());
                     annotations.put(NETWORK_ID, p.networkId());
                     annotations.put(DEVICE_ID, p.deviceId().toString());
-                    annotations.put(PORT_NUMBER, p.portNumber().toString());
                     annotations.put(IP_ADDRESS, p.ipAddress().toString());
                     annotations.put(MAC_ADDRESS, p.macAddress().toString());
 
+                    if (p.portNumber() != null) {
+                        annotations.put(PORT_NUMBER, p.portNumber().toString());
+                    }
+
                     client.pods().inNamespace(pod.getMetadata().getNamespace())
                             .withName(pod.getMetadata().getName())
                             .edit()
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
index 56fe815..15c184c 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
@@ -34,8 +34,6 @@
 import org.onosproject.k8snetworking.api.K8sPort;
 import org.onosproject.k8snetworking.util.RulePopulatorUtil;
 import org.onosproject.k8snode.api.K8sHost;
-import org.onosproject.k8snode.api.K8sHostEvent;
-import org.onosproject.k8snode.api.K8sHostListener;
 import org.onosproject.k8snode.api.K8sHostService;
 import org.onosproject.k8snode.api.K8sNode;
 import org.onosproject.k8snode.api.K8sNodeEvent;
@@ -72,7 +70,7 @@
 import static org.onosproject.k8snetworking.api.Constants.PRIORITY_DEFAULT_RULE;
 import static org.onosproject.k8snetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
 import static org.onosproject.k8snetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
-import static org.onosproject.k8snetworking.api.Constants.ROUTER_EXTRY_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.ROUTER_ENTRY_TABLE;
 import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
 import static org.onosproject.k8snetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
 import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
@@ -132,8 +130,6 @@
             new InternalK8sNetworkListener();
     private final InternalK8sNodeListener k8sNodeListener =
             new InternalK8sNodeListener();
-    private final InternalK8sHostListener k8sHostListener =
-            new InternalK8sHostListener();
     private final ExecutorService eventExecutor = newSingleThreadExecutor(
             groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
 
@@ -148,14 +144,12 @@
         leadershipService.runForLeadership(appId.name());
         k8sNetworkService.addListener(k8sNetworkListener);
         k8sNodeService.addListener(k8sNodeListener);
-        k8sHostService.addListener(k8sHostListener);
 
         log.info("Started");
     }
 
     @Deactivate
     protected void deactivate() {
-        k8sHostService.removeListener(k8sHostListener);
         k8sNodeService.removeListener(k8sNodeListener);
         k8sNetworkService.removeListener(k8sNetworkListener);
         leadershipService.withdraw(appId.name());
@@ -334,6 +328,9 @@
     private void setRouterSnatUpstreamRule(K8sNode k8sNode,
                                            K8sRouterBridge bridge,
                                            boolean install) {
+        if (k8sNode.routerPortNum() == null) {
+            return;
+        }
 
         TrafficSelector selector = DefaultTrafficSelector.builder()
                 .matchEthType(Ethernet.TYPE_IPV4)
@@ -350,13 +347,17 @@
                 selector,
                 treatment,
                 PRIORITY_DEFAULT_RULE,
-                ROUTER_EXTRY_TABLE,
+                ROUTER_ENTRY_TABLE,
                 install);
     }
 
     private void setRouterSnatDownstreamRule(K8sNode k8sNode,
                                              K8sRouterBridge bridge,
                                              boolean install) {
+        if (k8sNode.routerPortNum() == null) {
+            return;
+        }
+
         TrafficSelector selector = DefaultTrafficSelector.builder()
                 .matchEthType(Ethernet.TYPE_IPV4)
                 .matchInPort(k8sNode.routerPortNum())
@@ -373,10 +374,24 @@
                 selector,
                 treatment,
                 PRIORITY_DEFAULT_RULE,
-                ROUTER_EXTRY_TABLE,
+                ROUTER_ENTRY_TABLE,
                 install);
     }
 
+    private void setRouterSnatRules(K8sNode k8sNode, boolean install) {
+        for (K8sHost host : k8sHostService.completeHosts()) {
+            if (host.nodeNames().contains(k8sNode.hostname())) {
+                K8sRouterBridge bridge = host.routerBridges().stream()
+                        .filter(b -> b.segmentId() == k8sNode.segmentId())
+                        .findAny().orElse(null);
+                if (bridge != null) {
+                    setRouterSnatUpstreamRule(k8sNode, bridge, install);
+                    setRouterSnatDownstreamRule(k8sNode, bridge, install);
+                }
+            }
+        }
+    }
+
     private class InternalK8sNodeListener implements K8sNodeListener {
 
         private boolean isRelevantHelper() {
@@ -406,6 +421,7 @@
             setExtIntfArpRule(k8sNode, true);
             setExtSnatDownstreamRule(k8sNode, true);
             setContainerToExtRule(k8sNode, true);
+            setRouterSnatRules(k8sNode, true);
         }
 
         private void processNodeUpdate(K8sNode k8sNode) {
@@ -415,47 +431,6 @@
         }
     }
 
-    private class InternalK8sHostListener implements K8sHostListener {
-
-        private boolean isRelevantHelper() {
-            return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
-        }
-
-        @Override
-        public void event(K8sHostEvent event) {
-            switch (event.type()) {
-                case K8S_HOST_COMPLETE:
-                    eventExecutor.execute(() -> processNodeCompletion(event.subject()));
-                    break;
-                case K8S_HOST_INCOMPLETE:
-                default:
-                    break;
-            }
-        }
-
-        private void processNodeCompletion(K8sHost k8sHost) {
-            if (!isRelevantHelper()) {
-                return;
-            }
-
-            for (String name : k8sHost.nodeNames()) {
-                K8sNode node = k8sNodeService.node(name);
-                if (node == null) {
-                    return;
-                }
-                K8sRouterBridge bridge = k8sHost.routerBridges().stream()
-                        .filter(b -> b.segmentId() == node.segmentId())
-                        .findAny().orElse(null);
-                if (bridge == null) {
-                    return;
-                }
-
-                setRouterSnatUpstreamRule(node, bridge, true);
-                setRouterSnatDownstreamRule(node, bridge, true);
-            }
-        }
-    }
-
     private class InternalK8sNetworkListener implements K8sNetworkListener {
 
         private boolean isRelevantHelper() {
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
index 93d913d..45b592e 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
@@ -58,7 +58,6 @@
 import org.onosproject.k8snode.api.K8sNodeListener;
 import org.onosproject.k8snode.api.K8sNodeService;
 import org.onosproject.net.DeviceId;
-import org.onosproject.net.PortNumber;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.driver.DriverService;
 import org.onosproject.net.flow.DefaultTrafficSelector;
@@ -120,9 +119,7 @@
 import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.nodeIpGatewayIpMap;
 import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.podByIp;
 import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.portNumberByName;
-import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.tunnelPortNumByNetId;
 import static org.onosproject.k8snetworking.util.RulePopulatorUtil.CT_NAT_DST_FLAG;
-import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildExtension;
 import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildGroupBucket;
 import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildLoadExtension;
 import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildResubmitExtension;
@@ -643,34 +640,23 @@
                     tBuilder.setEthSrc(mac);
                 }
                 tBuilder.transition(STAT_EGRESS_TABLE);
-            } else {
-                K8sNode localNode = k8sNodeService.node(network.name());
-
-                tBuilder.setOutput(n.intgToTunPortNum());
-
-                PortNumber portNum = tunnelPortNumByNetId(network.networkId(),
-                        k8sNetworkService, n);
 
                 // install rules into tunnel bridge
-                TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder()
-                        .extension(buildExtension(
-                                deviceService,
-                                n.tunBridge(),
-                                localNode.dataIp().getIp4Address()),
-                                n.tunBridge())
-                        .setTunnelId(Long.valueOf(network.segmentId()))
-                        .setOutput(portNum)
+                TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+                        .setOutput(n.tunToIntgPortNum())
                         .build();
 
                 k8sFlowRuleService.setRule(
                         appId,
                         n.tunBridge(),
                         sBuilder.build(),
-                        treatmentToRemote,
+                        treatment,
                         PRIORITY_CIDR_RULE,
                         TUN_ENTRY_TABLE,
                         install
                 );
+            } else {
+                tBuilder.setOutput(n.intgToTunPortNum());
             }
 
             k8sFlowRuleService.setRule(
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
index f6c1d8b..50f3357 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
@@ -236,36 +236,43 @@
 
     private void processArpRequest(PacketContext context, Ethernet ethPacket) {
         ARP arpPacket = (ARP) ethPacket.getPayload();
-        K8sPort srcPort = k8sNetworkService.ports().stream()
+        K8sPort srcK8sPort = k8sNetworkService.ports().stream()
                 .filter(p -> p.macAddress().equals(ethPacket.getSourceMAC()))
                 .findAny().orElse(null);
 
-        if (srcPort == null && !context.inPacket().receivedFrom().port()
-                .equals(PortNumber.LOCAL)) {
+        PortNumber srcPortNum = context.inPacket().receivedFrom().port();
+        DeviceId srcDeviceId = context.inPacket().receivedFrom().deviceId();
+        boolean isEntryPort = false;
+
+        for (K8sNode node : k8sNodeService.completeNodes()) {
+            if (srcDeviceId.equals(node.intgBridge()) &&
+                    srcPortNum.equals(node.intgEntryPortNum())) {
+                isEntryPort = true;
+            }
+        }
+
+        // if the ARP request is not initiated from regular k8s ports nor
+        // integration bridge entry port, we simply ignore the ARP request...
+        if (srcK8sPort == null && !isEntryPort) {
             log.warn("Failed to find source port(MAC:{})", ethPacket.getSourceMAC());
             return;
         }
 
-        // FIXME: this is a workaround for storing host GW MAC address,
-        // need to find a way to store the MAC address in persistent way
-        if (context.inPacket().receivedFrom().port().equals(PortNumber.LOCAL)) {
-            gwMacAddress = ethPacket.getSourceMAC();
-        }
-
         IpAddress targetIp = Ip4Address.valueOf(arpPacket.getTargetProtocolAddress());
 
+        // look up the MAC address from regular k8s ports
         MacAddress replyMac = k8sNetworkService.ports().stream()
                 //        .filter(p -> p.networkId().equals(srcPort.networkId()))
                 .filter(p -> p.ipAddress().equals(targetIp))
                 .map(K8sPort::macAddress)
                 .findAny().orElse(null);
 
-        long gwIpCnt = k8sNetworkService.networks().stream()
-                .filter(n -> n.gatewayIp().equals(targetIp))
-                .count();
-
-        if (gwIpCnt > 0) {
-            replyMac = gwMacAddress;
+        // look up the MAC address from special integration entry port (e.g., LOCAL, k8s-int-os)
+        for (K8sNetwork network : k8sNetworkService.networks()) {
+            if (network.gatewayIp().equals(targetIp)) {
+                K8sNode node = k8sNodeService.node(network.name());
+                replyMac = node.intgEntryPortMac();
+            }
         }
 
         if (replyMac == null) {
@@ -334,10 +341,6 @@
         }
 
         if (replyMac == null) {
-            replyMac = MacAddress.valueOf(gatewayMac);
-        }
-
-        if (replyMac == null) {
             log.debug("Failed to find MAC address for {}", targetIp);
             return;
         }
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 3efe41b..8083e88 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -250,7 +250,7 @@
             }
 
             if (sameHost) {
-                TrafficSelector selector = DefaultTrafficSelector.builder()
+                TrafficSelector originalSelector = DefaultTrafficSelector.builder()
                         .matchEthType(Ethernet.TYPE_IPV4)
                         .matchIPSrc(IpPrefix.valueOf(srcNode.podCidr()))
                         .matchIPDst(IpPrefix.valueOf(dstNode.podCidr()))
@@ -263,7 +263,22 @@
                 k8sFlowRuleService.setRule(
                         appId,
                         dstNode.tunBridge(),
-                        selector,
+                        originalSelector,
+                        treatment,
+                        PRIORITY_INTER_NODE_RULE,
+                        TUN_ENTRY_TABLE,
+                        install);
+
+                TrafficSelector transformedSelector = DefaultTrafficSelector.builder()
+                        .matchEthType(Ethernet.TYPE_IPV4)
+                        .matchIPSrc(IpPrefix.valueOf(shiftIpDomain(srcNode.podCidr(), SHIFTED_IP_PREFIX)))
+                        .matchIPDst(IpPrefix.valueOf(dstNode.podCidr()))
+                        .build();
+
+                k8sFlowRuleService.setRule(
+                        appId,
+                        dstNode.tunBridge(),
+                        transformedSelector,
                         treatment,
                         PRIORITY_INTER_NODE_RULE,
                         TUN_ENTRY_TABLE,
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
index a0ccf5d..bec0540 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
@@ -36,6 +36,7 @@
 import org.onosproject.k8snode.api.K8sNodeListener;
 import org.onosproject.k8snode.api.K8sNodeService;
 import org.onosproject.mastership.MastershipService;
+import org.onosproject.net.DeviceId;
 import org.onosproject.net.PortNumber;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.driver.DriverService;
@@ -445,7 +446,12 @@
     private class InternalK8sNetworkListener implements K8sNetworkListener {
 
         private boolean isRelevantHelper(K8sNetworkEvent event) {
-            return mastershipService.isLocalMaster(event.port().deviceId());
+            DeviceId deviceId = event.port().deviceId();
+            if (deviceId == null) {
+                return false;
+            } else {
+                return mastershipService.isLocalMaster(deviceId);
+            }
         }
 
         @Override