apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeWatcher.java

/*
 * Copyright 2021-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.onosproject.kubevirtnode.impl;

import io.fabric8.kubernetes.api.model.Node;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.Watcher;
import io.fabric8.kubernetes.client.WatcherException;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.kubevirtnode.api.KubevirtApiConfig;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigEvent;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigListener;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeAdminService;
import org.onosproject.mastership.MastershipService;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.slf4j.Logger;

import java.util.Objects;
import java.util.concurrent.ExecutorService;

import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.MASTER;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
import static org.onosproject.kubevirtnode.api.KubevirtNodeService.APP_ID;
import static org.onosproject.kubevirtnode.api.KubevirtNodeState.INIT;
import static org.onosproject.kubevirtnode.util.KubevirtNodeUtil.buildKubevirtNode;
import static org.onosproject.kubevirtnode.util.KubevirtNodeUtil.k8sClient;
import static org.slf4j.LoggerFactory.getLogger;

/**
 * Kubernetes node watcher used for feeding node information.
 */
@Component(immediate = true)
public class KubevirtNodeWatcher {

    private final Logger log = getLogger(getClass());

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected CoreService coreService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected MastershipService mastershipService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected ClusterService clusterService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected LeadershipService leadershipService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected KubevirtNodeAdminService kubevirtNodeAdminService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected KubevirtApiConfigService kubevirtApiConfigService;

    private final ExecutorService eventExecutor = newSingleThreadExecutor(
            groupedThreads(this.getClass().getSimpleName(), "event-handler"));
    private final Watcher<Node> internalKubevirtNodeWatcher = new InternalKubevirtNodeWatcher();
    private final InternalKubevirtApiConfigListener
            internalKubevirtApiConfigListener = new InternalKubevirtApiConfigListener();

    private ApplicationId appId;
    private NodeId localNodeId;

    @Activate
    protected void activate() {
        appId = coreService.registerApplication(APP_ID);
        localNodeId = clusterService.getLocalNode().id();
        leadershipService.runForLeadership(appId.name());
        kubevirtApiConfigService.addListener(internalKubevirtApiConfigListener);

        log.info("Started");
    }

    @Deactivate
    protected void deactivate() {
        kubevirtApiConfigService.removeListener(internalKubevirtApiConfigListener);
        leadershipService.withdraw(appId.name());
        eventExecutor.shutdown();

        log.info("Stopped");
    }

    private void instantiateNodeWatcher() {
        KubevirtApiConfig config = kubevirtApiConfigService.apiConfig();
        if (config == null) {
            return;
        }
        KubernetesClient client = k8sClient(config);

        if (client != null) {
            client.nodes().watch(internalKubevirtNodeWatcher);
        }
    }

    private class InternalKubevirtApiConfigListener implements KubevirtApiConfigListener {

        private boolean isRelevantHelper() {
            return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
        }

        @Override
        public void event(KubevirtApiConfigEvent event) {

            switch (event.type()) {
                case KUBEVIRT_API_CONFIG_UPDATED:
                    eventExecutor.execute(this::processConfigUpdating);
                    break;
                case KUBEVIRT_API_CONFIG_CREATED:
                case KUBEVIRT_API_CONFIG_REMOVED:
                default:
                    // do nothing
                    break;
            }
        }

        private void processConfigUpdating() {
            if (!isRelevantHelper()) {
                return;
            }

            instantiateNodeWatcher();
        }
    }

    private class InternalKubevirtNodeWatcher implements Watcher<Node> {

        @Override
        public void eventReceived(Action action, Node node) {
            switch (action) {
                case ADDED:
                    eventExecutor.execute(() -> processAddition(node));
                    break;
                case MODIFIED:
                    eventExecutor.execute(() -> processModification(node));
                    break;
                case DELETED:
                    eventExecutor.execute(() -> processDeletion(node));
                    break;
                case ERROR:
                    log.warn("Failures processing node manipulation.");
                    break;
                default:
                    // do nothing
                    break;
            }
        }

        @Override
        public void onClose(WatcherException e) {
            // due to the bugs in fabric8, node watcher might be closed,
            // we will re-instantiate the node watcher in this case
            // FIXME: https://github.com/fabric8io/kubernetes-client/issues/2135
            log.warn("Node watcher OnClose, re-instantiate the node watcher...");
            instantiateNodeWatcher();
        }

        private void processAddition(Node node) {
            if (!isMaster()) {
                return;
            }

            log.trace("Process node {} creating event from API server.",
                    node.getMetadata().getName());

            KubevirtNode kubevirtNode = buildKubevirtNode(node);
            if (kubevirtNode.type() == WORKER || kubevirtNode.type() == GATEWAY) {
                if (!kubevirtNodeAdminService.hasNode(kubevirtNode.hostname())) {
                    kubevirtNodeAdminService.createNode(kubevirtNode);
                }
            }
        }

        private void processModification(Node node) {
            if (!isMaster()) {
                return;
            }

            log.trace("Process node {} updating event from API server.",
                    node.getMetadata().getName());

            KubevirtNode original = buildKubevirtNode(node);
            KubevirtNode existing = kubevirtNodeAdminService.node(node.getMetadata().getName());

            // if a master node is annotated as a gateway node, we simply add
            // the node into the cluster
            if (original.type() == GATEWAY && existing == null) {
                kubevirtNodeAdminService.createNode(original);
            }

            // if a gateway annotation removed from the master node, we simply remove
            // the node from the cluster
            if (original.type() == MASTER && existing != null && existing.type() == GATEWAY) {
                kubevirtNodeAdminService.removeNode(original.hostname());
            }

            if (existing != null) {
                // we update the kubevirt node and re-run bootstrapping,
                // if the updated node has different phyInts and data IP
                // this means we assume that the node's hostname, type and mgmt IP
                // are immutable
                if (!original.phyIntfs().equals(existing.phyIntfs()) ||
                        !original.dataIp().equals(existing.dataIp())) {
                    kubevirtNodeAdminService.updateNode(original.updateState(INIT));
                }
            }
        }

        private void processDeletion(Node node) {
            if (!isMaster()) {
                return;
            }

            log.trace("Process node {} removal event from API server.",
                    node.getMetadata().getName());

            KubevirtNode existing = kubevirtNodeAdminService.node(node.getMetadata().getName());

            if (existing != null) {
                kubevirtNodeAdminService.removeNode(node.getMetadata().getName());
            }
        }

        private boolean isMaster() {
            return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
        }
    }
}