DatabaseService subsystem: add admin commands, etc.
Change-Id: I24124579f5e0b03ccbf35a03230ae5a7aff95f22
diff --git a/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/ClusterMessagingProtocol.java b/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/ClusterMessagingProtocol.java
index f8243c7..6b7b7ff 100644
--- a/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/ClusterMessagingProtocol.java
+++ b/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/ClusterMessagingProtocol.java
@@ -169,7 +169,9 @@
@Override
public ProtocolClient createClient(TcpMember member) {
ControllerNode remoteNode = getControllerNode(member.host(), member.port());
- checkNotNull(remoteNode, "A valid controller node is expected");
+ checkNotNull(remoteNode,
+ "A valid controller node is expected for %s:%s",
+ member.host(), member.port());
return new ClusterMessagingProtocolClient(
clusterCommunicator, clusterService.getLocalNode(), remoteNode);
}
diff --git a/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/DatabaseManager.java b/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/DatabaseManager.java
index 6a84e04..f328842 100644
--- a/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/DatabaseManager.java
+++ b/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/DatabaseManager.java
@@ -2,21 +2,30 @@
import static org.slf4j.LoggerFactory.getLogger;
+import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import net.kuujo.copycat.Copycat;
import net.kuujo.copycat.StateMachine;
import net.kuujo.copycat.cluster.ClusterConfig;
+import net.kuujo.copycat.cluster.Member;
import net.kuujo.copycat.cluster.TcpCluster;
import net.kuujo.copycat.cluster.TcpClusterConfig;
import net.kuujo.copycat.cluster.TcpMember;
import net.kuujo.copycat.log.InMemoryLog;
import net.kuujo.copycat.log.Log;
+import org.apache.commons.lang3.RandomUtils;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
@@ -27,6 +36,8 @@
import org.onlab.onos.cluster.ClusterEventListener;
import org.onlab.onos.cluster.ClusterService;
import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.DefaultControllerNode;
+import org.onlab.onos.cluster.NodeId;
import org.onlab.onos.store.service.DatabaseAdminService;
import org.onlab.onos.store.service.DatabaseException;
import org.onlab.onos.store.service.DatabaseService;
@@ -38,8 +49,12 @@
import org.onlab.onos.store.service.WriteAborted;
import org.onlab.onos.store.service.WriteRequest;
import org.onlab.onos.store.service.WriteResult;
+import org.onlab.packet.IpAddress;
import org.slf4j.Logger;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
/**
* Strongly consistent and durable state management service based on
* Copycat implementation of Raft consensus protocol.
@@ -56,7 +71,19 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected DatabaseProtocolService copycatMessagingProtocol;
- public static final String LOG_FILE_PREFIX = "/tmp/onos-copy-cat-log";
+ public static final String LOG_FILE_PREFIX = "/tmp/onos-copy-cat-log_";
+
+ // Current working dir seems to be /opt/onos/apache-karaf-3.0.2
+ // TODO: Get the path to /opt/onos/config
+ private static final String CONFIG_DIR = "../config";
+
+ private static final String DEFAULT_MEMBER_FILE = "tablets.json";
+
+ private static final String DEFAULT_TABLET = "default";
+
+ // TODO: make this configurable
+ // initial member configuration file path
+ private String initialMemberConfig = DEFAULT_MEMBER_FILE;
private Copycat copycat;
private DatabaseClient client;
@@ -65,49 +92,75 @@
private ClusterConfig<TcpMember> clusterConfig;
private CountDownLatch clusterEventLatch;
-
private ClusterEventListener clusterEventListener;
+ private Map<String, Set<DefaultControllerNode>> tabletMembers;
+
+ private boolean autoAddMember = false;
+
@Activate
public void activate() {
// TODO: Not every node should be part of the consensus ring.
- final ControllerNode localNode = clusterService.getLocalNode();
- TcpMember localMember =
- new TcpMember(
- localNode.ip().toString(),
- localNode.tcpPort());
+ // load tablet configuration
+ File file = new File(CONFIG_DIR, initialMemberConfig);
+ log.info("Loading config: {}", file.getAbsolutePath());
+ TabletDefinitionStore tabletDef = new TabletDefinitionStore(file);
+ try {
+ tabletMembers = tabletDef.read();
+ } catch (IOException e) {
+ log.error("Failed to load tablet config {}", file);
+ throw new IllegalStateException("Failed to load tablet config", e);
+ }
+ // load default tablet configuration and start copycat
clusterConfig = new TcpClusterConfig();
- clusterConfig.setLocalMember(localMember);
+ Set<DefaultControllerNode> defaultMember = tabletMembers.get(DEFAULT_TABLET);
+ if (defaultMember == null || defaultMember.isEmpty()) {
+ log.error("No member found in [{}] tablet configuration.",
+ DEFAULT_TABLET);
+ throw new IllegalStateException("No member found in tablet configuration");
- List<TcpMember> remoteMembers = new ArrayList<>(clusterService.getNodes().size());
+ }
+ final ControllerNode localNode = clusterService.getLocalNode();
+ TcpMember clientHandler = null;
+ for (ControllerNode member : defaultMember) {
+ final TcpMember tcpMember = new TcpMember(member.ip().toString(),
+ member.tcpPort());
+ if (localNode.equals(member)) {
+ clientHandler = tcpMember;
+ clusterConfig.setLocalMember(tcpMember);
+ } else {
+ clusterConfig.addRemoteMember(tcpMember);
+ }
+ }
+
+ // TODO should be removed after DatabaseClient refactoring
+ if (clientHandler == null) {
+ Set<TcpMember> members = clusterConfig.getMembers();
+ if (members.isEmpty()) {
+ log.error("No member found in [{}] tablet configuration.",
+ DEFAULT_TABLET);
+ throw new IllegalStateException("No member found in tablet configuration");
+ }
+ int position = RandomUtils.nextInt(0, members.size());
+ clientHandler = Iterables.get(members, position);
+ }
+
+ // note: from this point beyond, clusterConfig requires synchronization
clusterEventLatch = new CountDownLatch(1);
clusterEventListener = new InternalClusterEventListener();
clusterService.addListener(clusterEventListener);
- // note: from this point beyond, clusterConfig requires synchronization
-
- for (ControllerNode node : clusterService.getNodes()) {
- TcpMember member = new TcpMember(node.ip().toString(), node.tcpPort());
- if (!member.equals(localMember)) {
- remoteMembers.add(member);
- }
- }
-
- if (remoteMembers.isEmpty()) {
- log.info("This node is the only node in the cluster. "
- + "Waiting for others to show up.");
- // FIXME: hack trying to relax cases forming multiple consensus rings.
- // add seed node configuration to avoid this
-
- // If the node is alone on it's own, wait some time
- // hoping other will come up soon
+ if (clusterService.getNodes().size() < clusterConfig.getMembers().size()) {
+ // current cluster size smaller then expected
try {
if (!clusterEventLatch.await(120, TimeUnit.SECONDS)) {
- log.info("Starting as single node cluster");
+ log.info("Starting with {}/{} nodes cluster",
+ clusterService.getNodes().size(),
+ clusterConfig.getMembers().size());
}
} catch (InterruptedException e) {
log.info("Interrupted waiting for others", e);
@@ -116,8 +169,6 @@
final TcpCluster cluster;
synchronized (clusterConfig) {
- clusterConfig.addRemoteMembers(remoteMembers);
-
// Create the cluster.
cluster = new TcpCluster(clusterConfig);
}
@@ -131,7 +182,8 @@
copycat = new Copycat(stateMachine, consensusLog, cluster, copycatMessagingProtocol);
copycat.start();
- client = new DatabaseClient(copycatMessagingProtocol.createClient(localMember));
+ // FIXME Redo DatabaseClient. Needs fall back mechanism etc.
+ client = new DatabaseClient(copycatMessagingProtocol.createClient(clientHandler));
log.info("Started.");
}
@@ -233,22 +285,34 @@
final TcpMember tcpMember = new TcpMember(node.ip().toString(),
node.tcpPort());
- log.trace("{}", event);
switch (event.type()) {
case INSTANCE_ACTIVATED:
case INSTANCE_ADDED:
- log.info("{} was added to the cluster", tcpMember);
- synchronized (clusterConfig) {
- clusterConfig.addRemoteMember(tcpMember);
+ if (autoAddMember) {
+ synchronized (clusterConfig) {
+ if (!clusterConfig.getMembers().contains(tcpMember)) {
+ log.info("{} was automatically added to the cluster", tcpMember);
+ clusterConfig.addRemoteMember(tcpMember);
+ }
+ }
}
break;
case INSTANCE_DEACTIVATED:
case INSTANCE_REMOVED:
- // FIXME to be replaced with admin interface
-// log.info("{} was removed from the cluster", tcpMember);
-// synchronized (clusterConfig) {
-// clusterConfig.removeRemoteMember(tcpMember);
-// }
+ if (autoAddMember) {
+ Set<DefaultControllerNode> members
+ = tabletMembers.getOrDefault(DEFAULT_TABLET,
+ Collections.emptySet());
+ // remove only if not the initial members
+ if (!members.contains(node)) {
+ synchronized (clusterConfig) {
+ if (clusterConfig.getMembers().contains(tcpMember)) {
+ log.info("{} was automatically removed from the cluster", tcpMember);
+ clusterConfig.removeRemoteMember(tcpMember);
+ }
+ }
+ }
+ }
break;
default:
break;
@@ -307,4 +371,58 @@
}
}
}
+
+ @Override
+ public void addMember(final ControllerNode node) {
+ final TcpMember tcpMember = new TcpMember(node.ip().toString(),
+ node.tcpPort());
+ log.info("{} was added to the cluster", tcpMember);
+ synchronized (clusterConfig) {
+ clusterConfig.addRemoteMember(tcpMember);
+ }
+ }
+
+ @Override
+ public void removeMember(final ControllerNode node) {
+ final TcpMember tcpMember = new TcpMember(node.ip().toString(),
+ node.tcpPort());
+ log.info("{} was removed from the cluster", tcpMember);
+ synchronized (clusterConfig) {
+ clusterConfig.removeRemoteMember(tcpMember);
+ }
+ }
+
+ @Override
+ public Collection<ControllerNode> listMembers() {
+ if (copycat == null) {
+ return ImmutableList.of();
+ }
+ Set<ControllerNode> members = new HashSet<>();
+ for (Member member : copycat.cluster().members()) {
+ if (member instanceof TcpMember) {
+ final TcpMember tcpMember = (TcpMember) member;
+ // TODO assuming tcpMember#host to be IP address,
+ // but if not lookup DNS, etc. first
+ IpAddress ip = IpAddress.valueOf(tcpMember.host());
+ int tcpPort = tcpMember.port();
+ NodeId id = getNodeIdFromIp(ip, tcpPort);
+ if (id == null) {
+ log.info("No NodeId found for {}:{}", ip, tcpPort);
+ continue;
+ }
+ members.add(new DefaultControllerNode(id, ip, tcpPort));
+ }
+ }
+ return members;
+ }
+
+ private NodeId getNodeIdFromIp(IpAddress ip, int tcpPort) {
+ for (ControllerNode node : clusterService.getNodes()) {
+ if (node.ip().equals(ip) &&
+ node.tcpPort() == tcpPort) {
+ return node.id();
+ }
+ }
+ return null;
+ }
}
diff --git a/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/TabletDefinitionStore.java b/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/TabletDefinitionStore.java
new file mode 100644
index 0000000..6d3d45e
--- /dev/null
+++ b/core/store/dist/src/main/java/org/onlab/onos/store/service/impl/TabletDefinitionStore.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2014 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onlab.onos.store.service.impl;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.onlab.onos.cluster.DefaultControllerNode;
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.packet.IpAddress;
+import org.slf4j.Logger;
+
+import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * Allows for reading and writing tablet definition as a JSON file.
+ */
+public class TabletDefinitionStore {
+
+ private final Logger log = getLogger(getClass());
+
+ private final File file;
+
+ /**
+ * Creates a reader/writer of the tablet definition file.
+ *
+ * @param filePath location of the definition file
+ */
+ public TabletDefinitionStore(String filePath) {
+ file = new File(filePath);
+ }
+
+ /**
+ * Creates a reader/writer of the tablet definition file.
+ *
+ * @param filePath location of the definition file
+ */
+ public TabletDefinitionStore(File filePath) {
+ file = checkNotNull(filePath);
+ }
+
+ /**
+ * Returns the Map from tablet name to set of initial member nodes.
+ *
+ * @return Map from tablet name to set of initial member nodes
+ * @throws IOException when I/O exception of some sort has occurred.
+ */
+ public Map<String, Set<DefaultControllerNode>> read() throws IOException {
+
+ final Map<String, Set<DefaultControllerNode>> tablets = new HashMap<>();
+
+ final ObjectMapper mapper = new ObjectMapper();
+ final ObjectNode tabletNodes = (ObjectNode) mapper.readTree(file);
+ final Iterator<Entry<String, JsonNode>> fields = tabletNodes.fields();
+ while (fields.hasNext()) {
+ final Entry<String, JsonNode> next = fields.next();
+ final Set<DefaultControllerNode> nodes = new HashSet<>();
+ final Iterator<JsonNode> elements = next.getValue().elements();
+ while (elements.hasNext()) {
+ ObjectNode nodeDef = (ObjectNode) elements.next();
+ nodes.add(new DefaultControllerNode(new NodeId(nodeDef.get("id").asText()),
+ IpAddress.valueOf(nodeDef.get("ip").asText()),
+ nodeDef.get("tcpPort").asInt(9876)));
+ }
+
+ tablets.put(next.getKey(), nodes);
+ }
+ return tablets;
+ }
+
+ /**
+ * Updates the Map from tablet name to set of member nodes.
+ *
+ * @param tabletName name of the tablet to update
+ * @param nodes set of initial member nodes
+ * @throws IOException when I/O exception of some sort has occurred.
+ */
+ public void write(String tabletName, Set<DefaultControllerNode> nodes) throws IOException {
+ checkNotNull(tabletName);
+ checkArgument(tabletName.isEmpty(), "Tablet name cannot be empty");
+ // TODO should validate if tabletName is allowed in JSON
+
+ // load current
+ Map<String, Set<DefaultControllerNode>> config;
+ try {
+ config = read();
+ } catch (IOException e) {
+ log.info("Reading tablet config failed, assuming empty definition.");
+ config = new HashMap<>();
+ }
+ // update with specified
+ config.put(tabletName, nodes);
+
+ // write back to file
+ final ObjectMapper mapper = new ObjectMapper();
+ final ObjectNode tabletNodes = mapper.createObjectNode();
+ for (Entry<String, Set<DefaultControllerNode>> tablet : config.entrySet()) {
+ ArrayNode nodeDefs = mapper.createArrayNode();
+ tabletNodes.set(tablet.getKey(), nodeDefs);
+
+ for (DefaultControllerNode node : tablet.getValue()) {
+ ObjectNode nodeDef = mapper.createObjectNode();
+ nodeDef.put("id", node.id().toString())
+ .put("ip", node.ip().toString())
+ .put("tcpPort", node.tcpPort());
+ nodeDefs.add(nodeDef);
+ }
+ }
+ mapper.writeTree(new JsonFactory().createGenerator(file, JsonEncoding.UTF8),
+ tabletNodes);
+ }
+
+}