Enhance RM logic in kubevirt networking app.

Change-Id: If12d7918486f6c8712004c276a2e84ede180c4dd
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
index c652d60..f879b6f 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
@@ -145,7 +145,7 @@
 
     @Override
     public int hashCode() {
-        return Objects.hash(serviceName, loadbalancerIp);
+        return Objects.hash(serviceName, loadbalancerIp.hashCode());
     }
 
     @Override
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
index 6061cd3..f6e956f 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
@@ -188,7 +188,7 @@
                 return;
             }
 
-            log.trace("processKubernetesExternalLbCreatedOrUpdated and updated elb with elecedGateway: {}", lb);
+            log.info("Create or update elb {}", lb);
 
             setExternalLbRulesForService(lb, true);
         }
@@ -204,7 +204,8 @@
                 return;
             }
 
-            log.trace("processKubernetesExternalLbGatewayChanged with oldateway: {}", oldGatway);
+            log.info("KubernetesExternalLbGatewayChanged from oldateway {} to new gateway {}",
+                    oldGatway, lb.electedGateway());
 
             setExternalLbRulesForService(lb.updateElectedGateway(oldGatway), false);
 
@@ -220,7 +221,8 @@
                 return;
             }
 
-            log.trace("processKubernetesExternalLbWorkerChanged with oldworker: {}", oldWorker);
+            log.info("ExternalLbWorkerChanged from oldworker {} to new worker {}",
+                    oldWorker, lb.electedWorker());
 
             setExternalLbRulesForService(lb.updateElectedWorker(oldWorker), false);
 
@@ -317,8 +319,9 @@
         }
 
         KubernetesExternalLbInterface externalLbInterface = gateway.kubernetesExternalLbInterface();
-        if (externalLbInterface == null) {
-            log.warn("setDownstreamRules called but externalLbInterface is null. Stop this task.");
+        if (externalLbInterface == null || externalLbInterface.externalLbGwMac() == null) {
+            log.warn("setDownstreamRules called but externalLbInterface is null or " +
+                    "externalLbInterfaceGwMac is null. Stop this task.");
             return;
         }
 
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
index a7d4dc5..d6af10f 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
@@ -18,14 +18,12 @@
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
-import io.fabric8.kubernetes.api.model.ConfigMap;
 import io.fabric8.kubernetes.api.model.LoadBalancerIngress;
 import io.fabric8.kubernetes.api.model.LoadBalancerStatus;
 import io.fabric8.kubernetes.api.model.Service;
 import io.fabric8.kubernetes.client.KubernetesClient;
 import io.fabric8.kubernetes.client.Watcher;
 import io.fabric8.kubernetes.client.WatcherException;
-import io.fabric8.kubernetes.client.dsl.Resource;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.MacAddress;
 import org.onosproject.cluster.ClusterService;
@@ -57,7 +55,6 @@
 import org.osgi.service.component.annotations.ReferenceCardinality;
 import org.slf4j.Logger;
 
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
@@ -308,7 +305,12 @@
         KubernetesClient client = k8sClient(apiConfigService);
 
         client.services().inNamespace(DEFAULT).list()
-                .getItems().forEach(this::addOrUpdateExternalLoadBalancer);
+                .getItems().forEach(service -> {
+                    if (addOrUpdateExternalLoadBalancer(service) &&
+                            !isLoadBalancerStatusAlreadySet(service)) {
+                        serviceStatusUpdate(service);
+                    }
+                });
     }
 
     private boolean addOrUpdateExternalLoadBalancer(Service service) {
@@ -431,7 +433,7 @@
             endpointSet.add(workerNode.dataIp().toString());
         });
 
-        String loadbalancerGatewayIp = loadBalancerGatewayIp();
+        IpAddress loadbalancerGatewayIp = loadBalancerGatewayIp();
 
         if (loadbalancerGatewayIp == null) {
             log.error("Can't find the loadbalancer gateway ip in the kubevip configmap.." +
@@ -451,28 +453,19 @@
                 .loadBalancerIp(IpAddress.valueOf(lbIp))
                 .servicePorts(servicePorts)
                 .endpointSet(endpointSet)
-                .loadBalancerGwIp(IpAddress.valueOf(loadbalancerGatewayIp))
+                .loadBalancerGwIp(loadbalancerGatewayIp)
                 .loadBalancerGwMac(loadBalancerGatewayMac)
                 .build();
     }
 
-    private String loadBalancerGatewayIp() {
-        KubernetesClient client = k8sClient(apiConfigService);
+    private IpAddress loadBalancerGatewayIp() {
+        KubernetesExternalLbConfig config = lbConfigService.lbConfigs().stream().findAny().orElse(null);
 
-        Resource<ConfigMap> kubeVipConfigMapResource =
-                client.configMaps().inNamespace(KUBE_SYSTEM).withName(KUBE_VIP);
-
-        if (kubeVipConfigMapResource == null) {
+        if (config == null) {
             return null;
         }
 
-        Map<String, String> kubeVipConfigMap = kubeVipConfigMapResource.get().getData();
-
-        if (!kubeVipConfigMap.containsKey(GATEWAY_IP)) {
-            return null;
-        }
-
-        return kubeVipConfigMap.get(GATEWAY_IP);
+        return config.loadBalancerGwIp();
     }
 
     private MacAddress loadBalancerGatewayMac() {
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
index 345e670..8b95507 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
@@ -542,7 +542,8 @@
         if (numOfGateways == 0) {
             return null;
         }
-        return (KubevirtNode) nodeService.completeNodes(GATEWAY).toArray()[router.hashCode() % numOfGateways];
+        return (KubevirtNode) nodeService.completeNodes(GATEWAY)
+                .toArray()[Math.floorMod(router.hashCode(), numOfGateways)];
     }
 
     /**
@@ -557,14 +558,18 @@
     public static KubevirtNode gatewayNodeForSpecifiedService(KubevirtNodeService nodeService,
                                                               KubernetesExternalLb externalLb) {
         //TODO: enhance election logic for a better load balancing
+        try {
+            int numOfGateways = nodeService.completeExternalLbGatewayNodes().size();
+            if (numOfGateways == 0) {
+                return null;
+            }
 
-        int numOfGateways = nodeService.completeExternalLbGatewayNodes().size();
-        if (numOfGateways == 0) {
+            return (KubevirtNode) nodeService.completeExternalLbGatewayNodes()
+                    .toArray()[Math.floorMod(externalLb.hashCode(), numOfGateways)];
+        } catch (IndexOutOfBoundsException e) {
+            log.error("IndexOutOfBoundsException occurred {}", e.toString());
             return null;
         }
-
-        return (KubevirtNode) nodeService.completeExternalLbGatewayNodes()
-                .toArray()[externalLb.hashCode() % numOfGateways];
     }
 
     /**
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbInterfaceCodec.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbInterfaceCodec.java
index cdd4402..838637b 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbInterfaceCodec.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbInterfaceCodec.java
@@ -48,9 +48,11 @@
         ObjectNode result = context.mapper().createObjectNode()
                 .put(ELB_BRIDGE_NAME, externalLbInterface.externalLbBridgeName())
                 .put(ELB_IP, externalLbInterface.externalLbIp().toString())
-                .put(ELB_GW_IP, externalLbInterface.externalLbGwIp().toString())
-                .put(ELB_GW_MAC, externalLbInterface.externalLbGwMac().toString());
+                .put(ELB_GW_IP, externalLbInterface.externalLbGwIp().toString());
 
+        if (externalLbInterface.externalLbGwMac() != null) {
+            result.put(ELB_GW_MAC, externalLbInterface.externalLbGwMac().toString());
+        }
         return result;
     }
 
@@ -69,16 +71,16 @@
         String elbGwIp = nullIsIllegal(json.get(ELB_GW_IP).asText(),
                 ELB_GW_IP + MISSING_MESSAGE);
 
-        String elbGwMac = nullIsIllegal(json.get(ELB_GW_MAC).asText(),
-                ELB_GW_MAC + MISSING_MESSAGE);
-
-        KubernetesExternalLbInterface externalLbInterface = DefaultKubernetesExternalLbInterface.builder()
+        KubernetesExternalLbInterface.Builder externalLbInterfaceBuilder = DefaultKubernetesExternalLbInterface
+                .builder()
                 .externalLbBridgeName(elbBridgeName)
                 .externallbGwIp(IpAddress.valueOf(elbGwIp))
-                .externalLbIp(IpAddress.valueOf(elbIp))
-                .externalLbGwMac(MacAddress.valueOf(elbGwMac))
-                .build();
+                .externalLbIp(IpAddress.valueOf(elbIp));
 
-        return externalLbInterface;
+        if (json.get(ELB_GW_MAC) != null) {
+            externalLbInterfaceBuilder.externalLbGwMac(MacAddress.valueOf(json.get(ELB_GW_MAC).asText()));
+        }
+
+        return externalLbInterfaceBuilder.build();
     }
 }
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
index 0850ba2..6626b74 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
@@ -1070,12 +1070,18 @@
                 node.phyIntfs().stream()
                         .filter(pi -> pi.intf().equals(portName))
                         .findAny()
-                        .ifPresent(pi -> setState(node, INCOMPLETE));
+                        .ifPresent(pi -> {
+                            log.info("Interface {} is down so set node {}'s state to INCOMPLETE",
+                                    pi.intf(),
+                                    node.hostname());
+                            setState(node, INCOMPLETE);
+                        });
             }
 
             //When the physical port up again, we set the node state to INIT
             //so that respective handlers do their related jobs.
-            if (node.state() == INCOMPLETE && node.type().equals(GATEWAY) && port.isEnabled()) {
+            if ((node.state() == INCOMPLETE || node.state() == DEVICE_CREATED)
+                    && node.type().equals(GATEWAY) && port.isEnabled()) {
                 node.phyIntfs().stream()
                         .filter(pi -> pi.intf().equals(portName))
                         .findAny()