apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java

/*
 * Copyright 2019-present Open Networking Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.onosproject.k8snode.impl;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import org.onlab.util.Tools;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.event.ListenerRegistry;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNode.Type;
import org.onosproject.k8snode.api.K8sNodeAdminService;
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
import org.onosproject.k8snode.api.K8sNodeStore;
import org.onosproject.k8snode.api.K8sNodeStoreDelegate;
import org.onosproject.net.DeviceId;
import org.onosproject.net.device.DeviceService;
import org.onosproject.store.service.AtomicCounter;
import org.onosproject.store.service.StorageService;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
import org.osgi.service.component.annotations.Modified;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.slf4j.Logger;

import java.util.Dictionary;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.k8snode.api.K8sApiConfig.Mode.NORMAL;
import static org.onosproject.k8snode.api.K8sNodeState.COMPLETE;
import static org.onosproject.k8snode.impl.OsgiPropertyConstants.OVSDB_PORT;
import static org.onosproject.k8snode.impl.OsgiPropertyConstants.OVSDB_PORT_NUM_DEFAULT;
import static org.onosproject.k8snode.util.K8sNodeUtil.genDpid;
import static org.slf4j.LoggerFactory.getLogger;

/**
 * Service administering the inventory of kubernetes nodes.
 */
@Component(
        immediate = true,
        service = { K8sNodeService.class, K8sNodeAdminService.class },
        property = {
                OVSDB_PORT + ":Integer=" + OVSDB_PORT_NUM_DEFAULT
        }
)
public class K8sNodeManager
        extends ListenerRegistry<K8sNodeEvent, K8sNodeListener>
        implements K8sNodeService, K8sNodeAdminService {

    private final Logger log = getLogger(getClass());

    private static final String MSG_NODE = "Kubernetes node %s %s";
    private static final String MSG_CREATED = "created";
    private static final String MSG_UPDATED = "updated";
    private static final String MSG_REMOVED = "removed";

    private static final String ERR_NULL_NODE = "Kubernetes node cannot be null";
    private static final String ERR_NULL_HOSTNAME = "Kubernetes node hostname cannot be null";
    private static final String ERR_NULL_DEVICE_ID = "Kubernetes node device ID cannot be null";

    private static final String DEVICE_ID_COUNTER_NAME = "device-id-counter";
    private static final String NOT_DUPLICATED_MSG = "% cannot be duplicated";

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected K8sNodeStore nodeStore;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected CoreService coreService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected ClusterService clusterService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected LeadershipService leadershipService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected StorageService storageService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY)
    protected DeviceService deviceService;

    /** OVSDB server listen port. */
    private int ovsdbPortNum = OVSDB_PORT_NUM_DEFAULT;

    private final ExecutorService eventExecutor = newSingleThreadExecutor(
            groupedThreads(this.getClass().getSimpleName(), "event-handler", log));

    private final K8sNodeStoreDelegate delegate = new K8sNodeManager.InternalNodeStoreDelegate();

    private AtomicCounter deviceIdCounter;

    private ApplicationId appId;

    @Activate
    protected void activate() {
        appId = coreService.registerApplication(APP_ID);
        nodeStore.setDelegate(delegate);

        leadershipService.runForLeadership(appId.name());

        deviceIdCounter = storageService.getAtomicCounter(DEVICE_ID_COUNTER_NAME);

        log.info("Started");
    }

    @Deactivate
    protected void deactivate() {
        nodeStore.unsetDelegate(delegate);

        leadershipService.withdraw(appId.name());
        eventExecutor.shutdown();

        log.info("Stopped");
    }

    @Modified
    protected void modified(ComponentContext context) {
        Dictionary<?, ?> properties = context.getProperties();
        int updatedOvsdbPort = Tools.getIntegerProperty(properties, OVSDB_PORT);
        if (!Objects.equals(updatedOvsdbPort, ovsdbPortNum)) {
            ovsdbPortNum = updatedOvsdbPort;
        }

        log.info("Modified");
    }

    @Override
    public void createNode(K8sNode node) {
        checkNotNull(node, ERR_NULL_NODE);

        K8sNode intNode;
        K8sNode extNode;
        K8sNode localNode;
        K8sNode tunNode;

        if (node.intgBridge() == null) {
            String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
            checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
            intNode = node.updateIntgBridge(DeviceId.deviceId(deviceIdStr));
            checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
                    NOT_DUPLICATED_MSG, intNode.intgBridge());
        } else {
            intNode = node;
            checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
                    NOT_DUPLICATED_MSG, intNode.intgBridge());
        }

        if (intNode.extBridge() == null) {
            String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
            checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
            extNode = intNode.updateExtBridge(DeviceId.deviceId(deviceIdStr));
            checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
                    NOT_DUPLICATED_MSG, extNode.extBridge());
        } else {
            extNode = intNode;
            checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
                    NOT_DUPLICATED_MSG, extNode.extBridge());
        }

        if (node.localBridge() == null) {
            String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
            checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
            localNode = extNode.updateLocalBridge(DeviceId.deviceId(deviceIdStr));
            checkArgument(!hasLocalBridge(localNode.localBridge(), localNode.hostname()),
                    NOT_DUPLICATED_MSG, localNode.localBridge());
        } else {
            localNode = extNode;
            checkArgument(!hasLocalBridge(localNode.localBridge(), localNode.hostname()),
                    NOT_DUPLICATED_MSG, localNode.localBridge());
        }

        if (node.mode() == NORMAL) {
            if (node.tunBridge() == null) {
                String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
                checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
                tunNode = localNode.updateTunBridge(DeviceId.deviceId(deviceIdStr));
                checkArgument(!hasTunBridge(tunNode.tunBridge(), tunNode.hostname()),
                        NOT_DUPLICATED_MSG, tunNode.tunBridge());
            } else {
                tunNode = localNode;
                checkArgument(!hasTunBridge(tunNode.tunBridge(), tunNode.hostname()),
                        NOT_DUPLICATED_MSG, tunNode.tunBridge());
            }

            nodeStore.createNode(tunNode);
        } else {
            nodeStore.createNode(localNode);
        }

        log.info(String.format(MSG_NODE, extNode.hostname(), MSG_CREATED));
    }

    @Override
    public void updateNode(K8sNode node) {
        checkNotNull(node, ERR_NULL_NODE);

        K8sNode intNode;
        K8sNode extNode;
        K8sNode localNode;
        K8sNode tunNode;

        K8sNode existingNode = nodeStore.node(node.hostname());
        checkNotNull(existingNode, ERR_NULL_NODE);

        DeviceId existIntgBridge = nodeStore.node(node.hostname()).intgBridge();

        if (node.intgBridge() == null) {
            intNode = node.updateIntgBridge(existIntgBridge);
            checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
                    NOT_DUPLICATED_MSG, intNode.intgBridge());
        } else {
            intNode = node;
            checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
                    NOT_DUPLICATED_MSG, intNode.intgBridge());
        }

        DeviceId existExtBridge = nodeStore.node(node.hostname()).extBridge();

        if (intNode.extBridge() == null) {
            extNode = intNode.updateExtBridge(existExtBridge);
            checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
                    NOT_DUPLICATED_MSG, extNode.extBridge());
        } else {
            extNode = intNode;
            checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
                    NOT_DUPLICATED_MSG, extNode.extBridge());
        }

        DeviceId existLocalBridge = nodeStore.node(node.hostname()).localBridge();

        if (extNode.localBridge() == null) {
            localNode = extNode.updateLocalBridge(existLocalBridge);
            checkArgument(!hasLocalBridge(localNode.localBridge(), localNode.hostname()),
                    NOT_DUPLICATED_MSG, localNode.localBridge());
        } else {
            localNode = extNode;
            checkArgument(!hasLocalBridge(localNode.localBridge(), localNode.hostname()),
                    NOT_DUPLICATED_MSG, localNode.localBridge());
        }

        if (node.mode() == NORMAL) {
            DeviceId existTunBridge = nodeStore.node(node.hostname()).tunBridge();

            if (localNode.tunBridge() == null) {
                tunNode = localNode.updateTunBridge(existTunBridge);
                checkArgument(!hasTunBridge(tunNode.tunBridge(), tunNode.hostname()),
                        NOT_DUPLICATED_MSG, tunNode.tunBridge());
            } else {
                tunNode = localNode;
                checkArgument(!hasTunBridge(tunNode.tunBridge(), tunNode.hostname()),
                        NOT_DUPLICATED_MSG, tunNode.tunBridge());
            }
            nodeStore.updateNode(tunNode);
        } else {
            nodeStore.updateNode(localNode);
        }
        log.info(String.format(MSG_NODE, extNode.hostname(), MSG_UPDATED));
    }

    @Override
    public K8sNode removeNode(String hostname) {
        checkArgument(!Strings.isNullOrEmpty(hostname), ERR_NULL_HOSTNAME);
        K8sNode node = nodeStore.removeNode(hostname);
        log.info(String.format(MSG_NODE, hostname, MSG_REMOVED));
        return node;
    }

    @Override
    public Set<K8sNode> nodes() {
        return nodeStore.nodes();
    }

    @Override
    public Set<K8sNode> nodes(String clusterName) {
        return nodeStore.nodes().stream()
                .filter(n -> n.clusterName().equals(clusterName))
                .collect(Collectors.toSet());
    }

    @Override
    public Set<K8sNode> nodes(Type type) {
        Set<K8sNode> nodes = nodeStore.nodes().stream()
                .filter(node -> Objects.equals(node.type(), type))
                .collect(Collectors.toSet());
        return ImmutableSet.copyOf(nodes);
    }

    @Override
    public Set<K8sNode> completeNodes() {
        Set<K8sNode> nodes = nodeStore.nodes().stream()
                .filter(node -> node.state() == COMPLETE)
                .collect(Collectors.toSet());
        return ImmutableSet.copyOf(nodes);
    }

    @Override
    public Set<K8sNode> completeNodes(Type type) {
        Set<K8sNode> nodes = nodeStore.nodes().stream()
                .filter(node -> node.type() == type &&
                        node.state() == COMPLETE)
                .collect(Collectors.toSet());
        return ImmutableSet.copyOf(nodes);
    }

    @Override
    public K8sNode node(String hostname) {
        return nodeStore.node(hostname);
    }

    // TODO: need to differentiate integration bridge and external bridge
    @Override
    public K8sNode node(DeviceId deviceId) {
        return nodeStore.nodes().stream()
                .filter(node -> Objects.equals(node.intgBridge(), deviceId) ||
                        Objects.equals(node.ovsdb(), deviceId))
                .findFirst().orElse(null);
    }

    private boolean hasIntgBridge(DeviceId deviceId, String hostname) {
        Optional<K8sNode> existNode = nodeStore.nodes().stream()
                .filter(n -> !n.hostname().equals(hostname))
                .filter(n -> deviceId.equals(n.intgBridge()))
                .findFirst();

        return existNode.isPresent();
    }

    private boolean hasExtBridge(DeviceId deviceId, String hostname) {
        Optional<K8sNode> existNode = nodeStore.nodes().stream()
                .filter(n -> !n.hostname().equals(hostname))
                .filter(n -> deviceId.equals(n.extBridge()))
                .findFirst();

        return existNode.isPresent();
    }

    private boolean hasLocalBridge(DeviceId deviceId, String hostname) {
        Optional<K8sNode> existNode = nodeStore.nodes().stream()
                .filter(n -> !n.hostname().equals(hostname))
                .filter(n -> deviceId.equals(n.localBridge()))
                .findFirst();

        return existNode.isPresent();
    }

    private boolean hasTunBridge(DeviceId deviceId, String hostname) {
        Optional<K8sNode> existNode = nodeStore.nodes().stream()
                .filter(n -> !n.hostname().equals(hostname))
                .filter(n -> deviceId.equals(n.tunBridge()))
                .findFirst();

        return existNode.isPresent();
    }

    private class InternalNodeStoreDelegate implements K8sNodeStoreDelegate {

        @Override
        public void notify(K8sNodeEvent event) {
            if (event != null) {
                log.trace("send kubernetes node event {}", event);
                process(event);
            }
        }
    }
}