Preperation for ZooKeeper failover
- Add template for specifying backup ZooKeeper connections.
- ONOS -> ZooKeeper
- RAMCloud -> ZooKeeper
- Fix RCClient to properly handle multiple ZooKeeper configuration.
Change-Id: I91ea078feca1b84fdf76a74624f8f5e0712214e8
diff --git a/conf/onos.properties b/conf/onos.properties
index 0235dc0..c8509cf 100644
--- a/conf/onos.properties
+++ b/conf/onos.properties
@@ -13,3 +13,5 @@
net.floodlightcontroller.forwarding.Forwarding.hardtimeout = 0
# NOTE: Do NOT modify or remove the line below. This value will be overwritten by onos.sh script.
net.onrc.onos.core.datagrid.HazelcastDatagrid.datagridConfig =
+# Uncomment and list all the ZooKeeper instances after localhost on multi-instance deployment.
+#net.onrc.onos.core.registry.ZookeeperRegistry.connectionString = localhost:2181,otherhost:2181
diff --git a/onos.sh b/onos.sh
index 32888c3..750c35a 100755
--- a/onos.sh
+++ b/onos.sh
@@ -374,6 +374,7 @@
# TODO make ZooKeeper address configurable.
echo "ramcloud.locator=zk:localhost:2181" > ${temp_rc}
+ echo "#ramcloud.locator=zk:localhost:2181,otherhost:2181" >> ${temp_rc}
echo "ramcloud.clusterName=${rc_cluster_name}" >> ${temp_rc}
end-conf-creation ${RAMCLOUD_CONF}
@@ -607,10 +608,14 @@
local coord_addr=`rc-coord-addr`
- # TODO Configuration for ZK address, port
- local zk_addr="localhost:2181"
+ # TODO Make ONOS_CONF readable from ONOS process then eliminate RAMCLOUD_CONF
+
+ # Configuration for ZK address, port
+ local rc_locator=$(read-conf ${RAMCLOUD_CONF} ramcloud.locator "zk:localhost:2181")
+
# RAMCloud cluster name
- local rc_cluster_name=$(read-conf ${ONOS_CONF} ramcloud.clusterName "ONOS-RC")
+ local rc_cluster_name=$(read-conf ${RAMCLOUD_CONF} ramcloud.clusterName "ONOS-RC")
+
# RAMCloud transport timeout
local rc_timeout=$(read-conf ${ONOS_CONF} ramcloud.timeout 1000)
# RAMCloud option deadServerTimeout
@@ -622,7 +627,7 @@
# (FYI: -C is documented to be deprecated in the document)
local coord_args="-C ${coord_addr}"
- coord_args="${coord_args} --externalStorage zk:${zk_addr}"
+ coord_args="${coord_args} --externalStorage ${rc_locator}"
coord_args="${coord_args} --clusterName ${rc_cluster_name}"
coord_args="${coord_args} --timeout ${rc_timeout}"
coord_args="${coord_args} --deadServerTimeout ${rc_coord_deadServerTimeout}"
@@ -656,10 +661,10 @@
local coord_addr=`rc-coord-addr`
- # TODO Configuration for ZK address, port
- local zk_addr="localhost:2181"
+ # Configuration for ZK address, port
+ local rc_locator=$(read-conf ${RAMCLOUD_CONF} ramcloud.locator "zk:localhost:2181")
# RAMCloud cluster name
- local rc_cluster_name=$(read-conf ${ONOS_CONF} ramcloud.clusterName "ONOS-RC")
+ local rc_cluster_name=$(read-conf ${RAMCLOUD_CONF} ramcloud.clusterName "ONOS-RC")
# RAMCloud option deadServerTimeout
# (note RC default is 250ms, setting relaxed ONOS default to 1000ms)
local rc_coord_deadServerTimeout=$(read-conf ${ONOS_CONF} ramcloud.coordinator.deadServerTimeout 1000)
@@ -669,7 +674,7 @@
# (FYI: -C is documented to be deprecated in the document)
local coord_args="-C ${coord_addr}"
- coord_args="${coord_args} --externalStorage zk:${zk_addr}"
+ coord_args="${coord_args} --externalStorage ${rc_locator}"
coord_args="${coord_args} --clusterName ${rc_cluster_name}"
# Note: --reset will reset ZK stored info and start running as acoordinator.
@@ -738,10 +743,10 @@
local logCleanerThreads=$(read-conf ${ONOS_CONF} ramcloud.server.logCleanerThreads 1)
local detectFailures=$(read-conf ${ONOS_CONF} ramcloud.server.detectFailures 0)
- # TODO Configuration for ZK address, port
- local zk_addr="localhost:2181"
+ # Configuration for ZK address, port
+ local rc_locator=$(read-conf ${RAMCLOUD_CONF} ramcloud.locator "zk:localhost:2181")
# RAMCloud cluster name
- local rc_cluster_name=$(read-conf ${ONOS_CONF} ramcloud.clusterName "ONOS-RC")
+ local rc_cluster_name=$(read-conf ${RAMCLOUD_CONF} ramcloud.clusterName "ONOS-RC")
# RAMCloud transport timeout
local rc_timeout=$(read-conf ${ONOS_CONF} ramcloud.timeout 1000)
# replication factor (-r) config
@@ -751,7 +756,7 @@
mkdir -p `dirname ${rc_datafile}`
local server_args="-L ${server_addr}"
- server_args="${server_args} --externalStorage zk:${zk_addr}"
+ server_args="${server_args} --externalStorage ${rc_locator}"
server_args="${server_args} --clusterName ${rc_cluster_name}"
server_args="${server_args} --timeout ${rc_timeout}"
server_args="${server_args} --masterServiceThreads ${masterServiceThreads}"
diff --git a/src/main/java/net/onrc/onos/core/datastore/ramcloud/RCClient.java b/src/main/java/net/onrc/onos/core/datastore/ramcloud/RCClient.java
index ff80494..687b6dd 100644
--- a/src/main/java/net/onrc/onos/core/datastore/ramcloud/RCClient.java
+++ b/src/main/java/net/onrc/onos/core/datastore/ramcloud/RCClient.java
@@ -37,6 +37,9 @@
import edu.stanford.ramcloud.JRamCloud.RejectRulesException;
import edu.stanford.ramcloud.JRamCloud.TableEnumerator2;
+/**
+ * RAMCloud implementation of datastore IKVClient.
+ */
public class RCClient implements IKVClient {
private static final Logger log = LoggerFactory.getLogger(RCClient.class);
@@ -45,10 +48,14 @@
private static final String DEFAULT_CLUSTERNAME = "ONOS-RC";
private static final String DB_CONFIG_FILE = "conf/ramcloud.conf";
- public static final Configuration CONFIG = getConfiguration();
+ private static final Configuration CONFIG = getConfiguration();
+ private static final String CLUSTER_NAME = getClusterName(CONFIG);
+ private static final String LOCATOR = getLocator(CONFIG);
- // Value taken from RAMCloud's Status.h
// FIXME These constants should be defined by JRamCloud
+ /**
+ * Constant defined in RAMCloud's Status.h.
+ */
public static final int STATUS_OK = 0;
/**
@@ -78,7 +85,7 @@
private static final ThreadLocal<JRamCloud> TLS_RC_CLIENT = new ThreadLocal<JRamCloud>() {
@Override
protected JRamCloud initialValue() {
- return new JRamCloud(getLocator(CONFIG), getClusterName(CONFIG));
+ return new JRamCloud(LOCATOR, CLUSTER_NAME);
}
};
@@ -95,16 +102,41 @@
// Currently RCClient is state-less
private static final RCClient THE_INSTANCE = new RCClient();
+ /**
+ * Default constructor.
+ */
+ protected RCClient() {
+ log.info("locator: {}, cluster name: {}", LOCATOR, CLUSTER_NAME);
+ }
+
+ /**
+ * Gets a DataStoreClient implemented on RAMCloud.
+ *
+ * @return RCClient
+ */
public static RCClient getClient() {
return THE_INSTANCE;
}
- public static final Configuration getConfiguration() {
- final File configFile = new File(System.getProperty("ramcloud.config.path", DB_CONFIG_FILE));
+ /**
+ * Gets the {@link Configuration} instance.
+ *
+ * @return Configuration
+ */
+ private static final Configuration getConfiguration() {
+ final File configFile = new File(
+ System.getProperty("ramcloud.config.path",
+ DB_CONFIG_FILE));
return getConfiguration(configFile);
}
- public static final Configuration getConfiguration(final File configFile) {
+ /**
+ * Gets the {@link Configuration} instance from properties file.
+ *
+ * @param configFile properties file
+ * @return Configuration
+ */
+ private static final Configuration getConfiguration(final File configFile) {
if (configFile == null) {
throw new IllegalArgumentException("Need to specify a configuration file or storage directory");
}
@@ -114,13 +146,23 @@
}
try {
- return new PropertiesConfiguration(configFile);
+ PropertiesConfiguration conf = new PropertiesConfiguration();
+ // stop parsing commas in property value
+ conf.setDelimiterParsingDisabled(true);
+ conf.load(configFile);
+ return conf;
} catch (ConfigurationException e) {
throw new IllegalArgumentException("Could not load configuration at: " + configFile, e);
}
}
- public static String getLocator(final Configuration configuration) {
+ /**
+ * Gets the RAMCloud external storage locator from configuration file.
+ *
+ * @param configuration input
+ * @return RAMCloud external storage locator string
+ */
+ private static String getLocator(final Configuration configuration) {
final String locator = configuration.getString("ramcloud.locator");
if (locator != null) {
@@ -142,13 +184,15 @@
return coordinatorURL;
}
- public static String getClusterName(final Configuration configuration) {
- final String clusterName = configuration.getString("ramcloud.clusterName");
- if (clusterName != null) {
- return clusterName;
- }
-
- return DEFAULT_CLUSTERNAME;
+ /**
+ * Gets the RAMCloud clusterName from configuration file.
+ *
+ * @param configuration input
+ * @return RAMCloud clusterName
+ */
+ private static String getClusterName(final Configuration configuration) {
+ return configuration.getString("ramcloud.clusterName",
+ DEFAULT_CLUSTERNAME);
}
@Override