Support VLAN network in gateway node.

Change-Id: Ia18287bf072e47787ba71865e8f8c4cb032dc455
(cherry picked from commit 4063f40a0eb0639bb96e5e0ebf5f39954ab7ccae)
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
index 85b520c..3f5620e 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
@@ -88,4 +88,6 @@
     public static final int CLI_LABELS_LENGTH = 30;
     public static final int CLI_CONTAINERS_LENGTH = 30;
     public static final int CLI_MARGIN_LENGTH = 2;
+
+    public static final int PRIORITY_STATEFUL_SNAT_RULE = 40500;
 }
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtFlowRuleManager.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtFlowRuleManager.java
index fa5aeda..ffb73e2 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtFlowRuleManager.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtFlowRuleManager.java
@@ -75,6 +75,7 @@
 import static org.onosproject.kubevirtnetworking.impl.OsgiPropertyConstants.PROVIDER_NETWORK_ONLY;
 import static org.onosproject.kubevirtnetworking.impl.OsgiPropertyConstants.PROVIDER_NETWORK_ONLY_DEFAULT;
 import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getPropertyValueAsBoolean;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
 import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -350,7 +351,8 @@
 
         @Override
         public boolean isRelevant(KubevirtNodeEvent event) {
-            return event.subject().type().equals(WORKER);
+            return event.subject().type().equals(WORKER) ||
+                    event.subject().type().equals(GATEWAY);
         }
 
         private boolean isRelevantHelper() {
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
index 9def76f..18caff1 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
@@ -52,6 +52,7 @@
 import org.onosproject.net.behaviour.InterfaceConfig;
 import org.onosproject.net.behaviour.PatchDescription;
 import org.onosproject.net.device.DeviceAdminService;
+import org.onosproject.net.driver.DriverService;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
 import org.onosproject.net.flow.TrafficSelector;
@@ -75,6 +76,7 @@
 import static org.onlab.util.Tools.groupedThreads;
 import static org.onosproject.kubevirtnetworking.api.Constants.DEFAULT_GATEWAY_MAC;
 import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRE_FLAT_TABLE;
 import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
 import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_DHCP_RULE;
 import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FORWARDING_RULE;
@@ -95,6 +97,8 @@
 import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
 import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveIpSrcToDstExtension;
 import static org.onosproject.kubevirtnode.api.Constants.TUNNEL_BRIDGE;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
 import static org.slf4j.LoggerFactory.getLogger;
 
 /**
@@ -133,6 +137,9 @@
     @Reference(cardinality = ReferenceCardinality.MANDATORY)
     protected KubevirtFlowRuleService flowService;
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected DriverService driverService;
+
     private final KubevirtNetworkListener networkListener = new InternalNetworkEventListener();
     private final KubevirtNodeListener nodeListener = new InternalNodeEventListener();
 
@@ -274,8 +281,10 @@
 
         setDhcpRule(deviceId, true);
         setForwardingRule(deviceId, true);
-        setGatewayArpRule(node, network, true);
-        setGatewayIcmpRule(node, network, true);
+        setGatewayArpRule(network, TENANT_ARP_TABLE,
+                network.tenantDeviceId(node.hostname()), true);
+        setGatewayIcmpRule(network, TENANT_ICMP_TABLE,
+                network.tenantDeviceId(node.hostname()), true);
 
         log.info("Install default flow rules for tenant bridge {}", network.tenantBridgeName());
     }
@@ -318,8 +327,9 @@
                 install);
     }
 
-    private void setGatewayArpRule(KubevirtNode node, KubevirtNetwork network, boolean install) {
-        Device device = deviceService.getDevice(network.tenantDeviceId(node.hostname()));
+    private void setGatewayArpRule(KubevirtNetwork network,
+                                   int tableNum, DeviceId deviceId, boolean install) {
+        Device device = deviceService.getDevice(deviceId);
 
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
         sBuilder.matchEthType(EthType.EtherType.ARP.ethType().toShort())
@@ -342,13 +352,14 @@
                 sBuilder.build(),
                 tBuilder.build(),
                 PRIORITY_ARP_GATEWAY_RULE,
-                TENANT_ARP_TABLE,
+                tableNum,
                 install
         );
     }
 
-    private void setGatewayIcmpRule(KubevirtNode node, KubevirtNetwork network, boolean install) {
-        DeviceId deviceId = network.tenantDeviceId(node.hostname());
+    private void
+    setGatewayIcmpRule(KubevirtNetwork network,
+                       int tableNum, DeviceId deviceId, boolean install) {
         TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
                 .matchEthType(Ethernet.TYPE_IPV4)
                 .matchIPProtocol(IPv4.PROTOCOL_ICMP)
@@ -374,10 +385,36 @@
                 sBuilder.build(),
                 tBuilder.build(),
                 PRIORITY_ICMP_RULE,
-                TENANT_ICMP_TABLE,
+                tableNum,
                 install);
     }
 
+
+    private void initGatewayNodeBridge(KubevirtNetwork network, boolean install) {
+        KubevirtNode electedGateway = gatewayNodeForSpecifiedNetwork(network);
+        if (electedGateway == null) {
+            log.warn("There's no elected gateway for the network {}", network.name());
+        }
+
+        setGatewayArpRule(network, PRE_FLAT_TABLE, electedGateway.intgBridge(), install);
+        setGatewayIcmpRule(network, PRE_FLAT_TABLE, electedGateway.intgBridge(), install);
+    }
+
+    /**
+     * Returns the gateway node for the specified network.
+     * Among gateways, only one gateway would act as a gateway per network.
+     *
+     * @param network kubevirt network
+     * @return gateway node which would act as the gateway for the network
+     */
+    private KubevirtNode gatewayNodeForSpecifiedNetwork(KubevirtNetwork network) {
+        //TODO: would implement election logic for each network.
+        //TODO: would implement cleanup logic in case a gateway node is added
+        // and the election is changed
+        return nodeService.completeNodes(GATEWAY).stream()
+                .findFirst().orElse(null);
+    }
+
     private class InternalNetworkEventListener implements KubevirtNetworkListener {
 
         private boolean isRelevantHelper() {
@@ -413,6 +450,8 @@
                     break;
                 case FLAT:
                 case VLAN:
+                    initGatewayNodeBridge(network, true);
+                    break;
                 default:
                     // do nothing
                     break;
@@ -432,6 +471,8 @@
                     break;
                 case FLAT:
                 case VLAN:
+                    initGatewayNodeBridge(network, false);
+                    break;
                 default:
                     // do nothing
                     break;
@@ -487,23 +528,40 @@
                 return;
             }
 
-            for (KubevirtNetwork network : networkService.networks()) {
-                switch (network.type()) {
-                    case VXLAN:
-                    case GRE:
-                    case GENEVE:
-                        if (network.segmentId() == null) {
-                            continue;
-                        }
-                        createBridge(node, network);
-                        createPatchInterface(node, network);
-                        setDefaultRules(node, network);
-                        break;
-                    case FLAT:
-                    case VLAN:
-                    default:
-                        // do nothing
-                        break;
+            if (node.type().equals(WORKER)) {
+                for (KubevirtNetwork network : networkService.networks()) {
+                    switch (network.type()) {
+                        case VXLAN:
+                        case GRE:
+                        case GENEVE:
+                            if (network.segmentId() == null) {
+                                continue;
+                            }
+                            createBridge(node, network);
+                            createPatchInterface(node, network);
+                            setDefaultRules(node, network);
+                            break;
+                        case FLAT:
+                        case VLAN:
+                        default:
+                            // do nothing
+                            break;
+                    }
+                }
+            } else if (node.type().equals(GATEWAY)) {
+                for (KubevirtNetwork network : networkService.networks()) {
+                    switch (network.type()) {
+                        case FLAT:
+                        case VLAN:
+                            initGatewayNodeBridge(network, true);
+                            break;
+                        case VXLAN:
+                        case GRE:
+                        case GENEVE:
+                        default:
+                            // do nothing
+                            break;
+                    }
                 }
             }
         }
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/RulePopulatorUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/RulePopulatorUtil.java
index f005c5f..610066f 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/RulePopulatorUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/RulePopulatorUtil.java
@@ -16,14 +16,22 @@
 package org.onosproject.kubevirtnetworking.util;
 
 import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.TpPort;
 import org.onosproject.net.Device;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
 import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverHandler;
+import org.onosproject.net.driver.DriverService;
 import org.onosproject.net.flow.instructions.ExtensionPropertyException;
 import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatmentType;
 import org.slf4j.Logger;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_LOAD;
 import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_ARP_SHA_TO_THA;
 import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_ARP_SPA_TO_TPA;
@@ -47,6 +55,46 @@
     private static final String VALUE = "value";
     private static final String TUNNEL_DST = "tunnelDst";
 
+    private static final String CT_FLAGS = "flags";
+    private static final String CT_ZONE = "zone";
+    private static final String CT_TABLE = "recircTable";
+    private static final String CT_STATE = "ctState";
+    private static final String CT_STATE_MASK = "ctStateMask";
+    private static final String CT_PRESENT_FLAGS = "presentFlags";
+    private static final String CT_IPADDRESS_MIN = "ipAddressMin";
+    private static final String CT_IPADDRESS_MAX = "ipAddressMax";
+    private static final String CT_PORT_MIN = "portMin";
+    private static final String CT_PORT_MAX = "portMax";
+    private static final String CT_NESTED_ACTIONS = "nestedActions";
+
+    public static final int CT_NAT_SRC_FLAG = 0;
+    public static final int CT_NAT_DST_FLAG = 1;
+    public static final int CT_NAT_PERSISTENT_FLAG = 2;
+    public static final int CT_NAT_PROTO_HASH_FLAG = 3;
+    public static final int CT_NAT_PROTO_RANDOM_FLAG = 4;
+
+    private static final int ADDRESS_V4_MIN_FLAG = 0;
+    private static final int ADDRESS_V4_MAX_FLAG = 1;
+    private static final int ADDRESS_V6_MIN_FLAG = 2;
+    private static final int ADDRESS_V6_MAX_FLAG = 3;
+    private static final int PORT_MIN_FLAG = 4;
+    private static final int PORT_MAX_FLAG = 5;
+
+    private static final String STR_ZERO = "0";
+    private static final String STR_ONE = "1";
+    private static final String STR_PADDING = "0000000000000000";
+    private static final int MASK_BEGIN_IDX = 0;
+    private static final int MASK_MAX_IDX = 16;
+    private static final int MASK_RADIX = 2;
+    private static final int PORT_RADIX = 16;
+
+    // Refer to http://openvswitch.org/support/dist-docs/ovs-fields.7.txt for the values
+    public static final long CT_STATE_NONE = 0;
+    public static final long CT_STATE_NEW = 0x01;
+    public static final long CT_STATE_EST = 0x02;
+    public static final long CT_STATE_NOT_TRK = 0x20;
+    public static final long CT_STATE_TRK = 0x20;
+
     // layer 3 nicira fields
     private static final int NXM_OF_IP_SRC = 0x00000e04;
     private static final int NXM_OF_IP_DST = 0x00001004;
@@ -70,6 +118,7 @@
     public static final int NXM_OF_ICMP_TYPE = 0x00001a01;
     public static final int NXM_OF_ICMP_CODE = 0x00001c01;
 
+
     private RulePopulatorUtil() {
     }
 
@@ -207,4 +256,202 @@
             return null;
         }
     }
+
+    public static NiciraConnTrackTreatmentBuilder niciraConnTrackTreatmentBuilder(DriverService ds, DeviceId id) {
+        return new NiciraConnTrackTreatmentBuilder(ds, id);
+    }
+    /**
+     * Builder class for OVS Connection Tracking feature actions.
+     */
+    public static final class NiciraConnTrackTreatmentBuilder {
+
+        private DriverService driverService;
+        private DeviceId deviceId;
+        private IpAddress natAddress = null;
+        private TpPort natPortMin = null;
+        private TpPort natPortMax = null;
+        private int zone;
+        private boolean commit;
+        private short table = -1;
+        private boolean natAction;
+        private int natFlag;
+
+        // private constructor
+        private NiciraConnTrackTreatmentBuilder(DriverService driverService,
+                                                DeviceId deviceId) {
+            this.driverService = driverService;
+            this.deviceId = deviceId;
+        }
+
+        /**
+         * Sets commit flag.
+         *
+         * @param c true if commit, false if not.
+         * @return NiriraConnTrackTreatmentBuilder object
+         */
+        public NiciraConnTrackTreatmentBuilder commit(boolean c) {
+            this.commit = c;
+            return this;
+        }
+
+        /**
+         * Sets zone number.
+         *
+         * @param z zone number
+         * @return NiriraConnTrackTreatmentBuilder object
+         */
+        public NiciraConnTrackTreatmentBuilder zone(int z) {
+            this.zone = z;
+            return this;
+        }
+
+        /**
+         * Sets recirculation table number.
+         *
+         * @param t table number to restart
+         * @return NiriraConnTrackTreatmentBuilder object
+         */
+        public NiciraConnTrackTreatmentBuilder table(short t) {
+            this.table = t;
+            return this;
+        }
+
+        /**
+         * Sets IP address for NAT.
+         *
+         * @param ip NAT IP address
+         * @return NiriraConnTrackTreatmentBuilder object
+         */
+        public NiciraConnTrackTreatmentBuilder natIp(IpAddress ip) {
+            this.natAddress = ip;
+            return this;
+        }
+
+        /**
+         * Sets min port for NAT.
+         *
+         * @param port port number
+         * @return NiciraConnTrackTreatmentBuilder object
+         */
+        public NiciraConnTrackTreatmentBuilder natPortMin(TpPort port) {
+            this.natPortMin = port;
+            return this;
+        }
+
+        /**
+         * Sets max port for NAT.
+         *
+         * @param port port number
+         * @return NiciraConnTrackTreatmentBuilder object
+         */
+        public NiciraConnTrackTreatmentBuilder natPortMax(TpPort port) {
+            this.natPortMax = port;
+            return this;
+        }
+
+        /**
+         * Sets NAT flags.
+         * SRC NAT: 1 << 0
+         * DST NAT: 1 << 1
+         * PERSISTENT NAT: 1 << 2
+         * PROTO_HASH NAT: 1 << 3
+         * PROTO_RANDOM NAT : 1 << 4
+         *
+         * @param flag flag value
+         * @return NiciraConnTrackTreatmentBuilder object
+         */
+        public NiciraConnTrackTreatmentBuilder natFlag(int flag) {
+            this.natFlag = 1 << flag;
+            return this;
+        }
+
+        /**
+         * Sets the flag for NAT action.
+         *
+         * @param nat nat action is included if true, no nat action otherwise
+         * @return NiriraConnTrackTreatmentBuilder object
+         */
+        public NiciraConnTrackTreatmentBuilder natAction(boolean nat) {
+            this.natAction = nat;
+            return this;
+        }
+
+        /**
+         * Builds extension treatment for OVS ConnTack and NAT feature.
+         *
+         * @return ExtensionTreatment object
+         */
+        public ExtensionTreatment build() {
+            DriverHandler handler = driverService.createHandler(deviceId);
+            ExtensionTreatmentResolver etr =
+                    handler.behaviour(ExtensionTreatmentResolver.class);
+
+            ExtensionTreatment natTreatment = etr.getExtensionInstruction(
+                    ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_NAT.type());
+            try {
+
+                if (natAddress == null && natPortMin == null && natPortMax == null) {
+                    natTreatment.setPropertyValue(CT_FLAGS, 0);
+                    natTreatment.setPropertyValue(CT_PRESENT_FLAGS, 0);
+                } else {
+                    natTreatment.setPropertyValue(CT_FLAGS, this.natFlag);
+
+                    natTreatment.setPropertyValue(CT_PRESENT_FLAGS,
+                            buildPresentFlag((natPortMin != null && natPortMax != null),
+                                    natAddress != null));
+                }
+
+                if (natAddress != null) {
+                    natTreatment.setPropertyValue(CT_IPADDRESS_MIN, natAddress);
+                    natTreatment.setPropertyValue(CT_IPADDRESS_MAX, natAddress);
+                }
+
+                if (natPortMin != null) {
+                    natTreatment.setPropertyValue(CT_PORT_MIN, natPortMin.toInt());
+                }
+
+                if (natPortMax != null) {
+                    natTreatment.setPropertyValue(CT_PORT_MAX, natPortMax.toInt());
+                }
+
+            } catch (Exception e) {
+                log.error("Failed to set NAT due to error : {}", e);
+                return null;
+            }
+
+            ExtensionTreatment ctTreatment = etr.getExtensionInstruction(
+                    ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_CT.type());
+            try {
+                List<ExtensionTreatment> nat = new ArrayList<>();
+                if (natAction) {
+                    nat.add(natTreatment);
+                }
+                ctTreatment.setPropertyValue(CT_FLAGS, commit ? 1 : 0);
+                ctTreatment.setPropertyValue(CT_ZONE, zone);
+                ctTreatment.setPropertyValue(CT_TABLE, table > -1 ? table : 0xff);
+                ctTreatment.setPropertyValue(CT_NESTED_ACTIONS, nat);
+            } catch (Exception e) {
+                log.error("Failed to set CT due to error : {}", e);
+                return null;
+            }
+
+            return ctTreatment;
+        }
+
+        private int buildPresentFlag(boolean isPortPresent, boolean isAddressPresent) {
+
+            int presentFlag = 0;
+
+            if (isPortPresent) {
+                presentFlag = 1 << PORT_MIN_FLAG | 1 << PORT_MAX_FLAG;
+            }
+
+            if (isAddressPresent) {
+                // TODO: need to support IPv6 address
+                presentFlag =  presentFlag | 1 << ADDRESS_V4_MIN_FLAG | 1 << ADDRESS_V4_MAX_FLAG;
+            }
+
+            return presentFlag;
+        }
+    }
 }