blob: 8cd2d13b0d898f62bec3ed98930552a8fffa74c7 [file] [log] [blame]
Madan Jampaniad3c5262016-01-20 00:50:17 -08001/*
Brian O'Connora09fe5b2017-08-03 21:12:30 -07002 * Copyright 2016-present Open Networking Foundation
Madan Jampaniad3c5262016-01-20 00:50:17 -08003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package org.onosproject.cluster.impl;
17
Jordan Halterman00e92da2018-05-22 23:05:52 -070018import java.io.File;
19import java.io.FileInputStream;
20import java.io.IOException;
21import java.net.HttpURLConnection;
22import java.net.URL;
23import java.util.Set;
24import java.util.UUID;
25import java.util.concurrent.ScheduledExecutorService;
26import java.util.concurrent.TimeUnit;
27import java.util.concurrent.atomic.AtomicReference;
28import java.util.stream.Collectors;
29
Ray Milkey6a51cb92018-03-06 09:03:03 -080030import com.fasterxml.jackson.databind.ObjectMapper;
Jordan Halterman19c123a2018-07-30 13:57:19 -070031import com.google.common.collect.Sets;
Ray Milkey6a51cb92018-03-06 09:03:03 -080032import com.google.common.io.Files;
Ray Milkeyd84f89b2018-08-17 14:54:17 -070033
34import org.osgi.service.component.annotations.Activate;
35import org.osgi.service.component.annotations.Component;
36import org.osgi.service.component.annotations.Deactivate;
37import org.osgi.service.component.annotations.Reference;
38import org.osgi.service.component.annotations.ReferenceCardinality;
39import org.onlab.packet.IpAddress;
40
Madan Jampaniad3c5262016-01-20 00:50:17 -080041import org.onosproject.cluster.ClusterMetadata;
42import org.onosproject.cluster.ClusterMetadataProvider;
43import org.onosproject.cluster.ClusterMetadataProviderRegistry;
44import org.onosproject.cluster.ClusterMetadataProviderService;
Madan Jampaniad3c5262016-01-20 00:50:17 -080045import org.onosproject.cluster.DefaultControllerNode;
Jordan Halterman00e92da2018-05-22 23:05:52 -070046import org.onosproject.cluster.Node;
Madan Jampaniad3c5262016-01-20 00:50:17 -080047import org.onosproject.cluster.NodeId;
Madan Jampaniad3c5262016-01-20 00:50:17 -080048import org.onosproject.cluster.PartitionId;
49import org.onosproject.net.provider.ProviderId;
50import org.onosproject.store.service.Versioned;
51import org.slf4j.Logger;
52
Madan Jampaniad3c5262016-01-20 00:50:17 -080053import static com.google.common.base.Preconditions.checkState;
Yuta HIGUCHI1624df12016-07-21 16:54:33 -070054import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
Ray Milkey6a51cb92018-03-06 09:03:03 -080055import static org.onlab.util.Tools.groupedThreads;
56import static org.slf4j.LoggerFactory.getLogger;
Madan Jampaniad3c5262016-01-20 00:50:17 -080057
58/**
59 * Provider of {@link ClusterMetadata cluster metadata} sourced from a local config file.
60 */
61@Component(immediate = true)
62public class ConfigFileBasedClusterMetadataProvider implements ClusterMetadataProvider {
63
64 private final Logger log = getLogger(getClass());
65
Madan Jampani898fcca2016-02-26 12:42:08 -080066 private static final String CONFIG_DIR = "../config";
67 private static final String CONFIG_FILE_NAME = "cluster.json";
68 private static final File CONFIG_FILE = new File(CONFIG_DIR, CONFIG_FILE_NAME);
Madan Jampaniad3c5262016-01-20 00:50:17 -080069
Ray Milkeyd84f89b2018-08-17 14:54:17 -070070 @Reference(cardinality = ReferenceCardinality.MANDATORY)
Madan Jampaniad3c5262016-01-20 00:50:17 -080071 protected ClusterMetadataProviderRegistry providerRegistry;
72
Madan Jampanif172d402016-03-04 00:56:38 -080073 private static final ProviderId PROVIDER_ID = new ProviderId("file", "none");
Madan Jampani898fcca2016-02-26 12:42:08 -080074 private final AtomicReference<Versioned<ClusterMetadata>> cachedMetadata = new AtomicReference<>();
Madan Jampanif172d402016-03-04 00:56:38 -080075 private final ScheduledExecutorService configFileChangeDetector =
Yuta HIGUCHI1624df12016-07-21 16:54:33 -070076 newSingleThreadScheduledExecutor(groupedThreads("onos/cluster/metadata/config-watcher", "", log));
Madan Jampaniad3c5262016-01-20 00:50:17 -080077
Madan Jampanif172d402016-03-04 00:56:38 -080078 private String metadataUrl;
Madan Jampaniad3c5262016-01-20 00:50:17 -080079 private ObjectMapper mapper;
80 private ClusterMetadataProviderService providerService;
81
82 @Activate
83 public void activate() {
84 mapper = new ObjectMapper();
Madan Jampaniad3c5262016-01-20 00:50:17 -080085 providerService = providerRegistry.register(this);
Madan Jampanif172d402016-03-04 00:56:38 -080086 metadataUrl = System.getProperty("onos.cluster.metadata.uri", "file://" + CONFIG_DIR + "/" + CONFIG_FILE);
87 configFileChangeDetector.scheduleWithFixedDelay(() -> watchUrl(metadataUrl), 100, 500, TimeUnit.MILLISECONDS);
Madan Jampaniad3c5262016-01-20 00:50:17 -080088 log.info("Started");
89 }
90
91 @Deactivate
92 public void deactivate() {
Madan Jampani898fcca2016-02-26 12:42:08 -080093 configFileChangeDetector.shutdown();
Madan Jampaniad3c5262016-01-20 00:50:17 -080094 providerRegistry.unregister(this);
95 log.info("Stopped");
96 }
97
98 @Override
99 public ProviderId id() {
100 return PROVIDER_ID;
101 }
102
103 @Override
104 public Versioned<ClusterMetadata> getClusterMetadata() {
105 checkState(isAvailable());
106 synchronized (this) {
107 if (cachedMetadata.get() == null) {
Jordan Haltermandbfff062017-06-19 10:31:32 -0700108 cachedMetadata.set(blockForMetadata(metadataUrl));
Madan Jampaniad3c5262016-01-20 00:50:17 -0800109 }
110 return cachedMetadata.get();
111 }
112 }
113
Jordan Halterman00e92da2018-05-22 23:05:52 -0700114 private ClusterMetadataPrototype toPrototype(ClusterMetadata metadata) {
115 ClusterMetadataPrototype prototype = new ClusterMetadataPrototype();
116 prototype.setName(metadata.getName());
Jordan Halterman19c123a2018-07-30 13:57:19 -0700117 prototype.setController(metadata.getNodes()
118 .stream()
119 .map(this::toPrototype)
120 .collect(Collectors.toSet()));
121 prototype.setStorage(metadata.getStorageNodes()
122 .stream()
123 .map(this::toPrototype)
124 .collect(Collectors.toSet()));
Jordan Halterman00e92da2018-05-22 23:05:52 -0700125 return prototype;
126 }
127
128 private NodePrototype toPrototype(Node node) {
129 NodePrototype prototype = new NodePrototype();
130 prototype.setId(node.id().id());
Jordan Haltermane458f002018-09-18 17:42:05 -0700131 prototype.setHost(node.host());
Jordan Halterman00e92da2018-05-22 23:05:52 -0700132 prototype.setPort(node.tcpPort());
133 return prototype;
134 }
135
Madan Jampaniad3c5262016-01-20 00:50:17 -0800136 @Override
137 public void setClusterMetadata(ClusterMetadata metadata) {
138 try {
Ray Milkey83200882017-11-13 19:18:21 -0800139 File configFile = new File(metadataUrl.replaceFirst("file://", ""));
140 Files.createParentDirs(configFile);
Jordan Halterman00e92da2018-05-22 23:05:52 -0700141 ClusterMetadataPrototype metadataPrototype = toPrototype(metadata);
142 mapper.writeValue(configFile, metadataPrototype);
Ray Milkey83200882017-11-13 19:18:21 -0800143 cachedMetadata.set(fetchMetadata(metadataUrl));
144 providerService.clusterMetadataChanged(new Versioned<>(metadata, configFile.lastModified()));
Madan Jampaniad3c5262016-01-20 00:50:17 -0800145 } catch (IOException e) {
Ray Milkey6a51cb92018-03-06 09:03:03 -0800146 throw new IllegalArgumentException(e);
Madan Jampaniad3c5262016-01-20 00:50:17 -0800147 }
148 }
149
150 @Override
151 public void addActivePartitionMember(PartitionId partitionId, NodeId nodeId) {
152 throw new UnsupportedOperationException();
153 }
154
155 @Override
156 public void removeActivePartitionMember(PartitionId partitionId, NodeId nodeId) {
157 throw new UnsupportedOperationException();
158 }
159
160 @Override
161 public Set<NodeId> getActivePartitionMembers(PartitionId partitionId) {
162 throw new UnsupportedOperationException();
163 }
164
165 @Override
166 public boolean isAvailable() {
Madan Jampanif172d402016-03-04 00:56:38 -0800167 try {
168 URL url = new URL(metadataUrl);
Jon Halla3fcf672017-03-28 16:53:22 -0700169 if ("file".equals(url.getProtocol())) {
Madan Jampanif172d402016-03-04 00:56:38 -0800170 File file = new File(metadataUrl.replaceFirst("file://", ""));
171 return file.exists();
Madan Jampanif172d402016-03-04 00:56:38 -0800172 } else {
Jordan Halterman46c76902017-08-24 14:55:25 -0700173 // Return true for HTTP URLs since we allow blocking until HTTP servers come up
174 return "http".equals(url.getProtocol());
Madan Jampanif172d402016-03-04 00:56:38 -0800175 }
176 } catch (Exception e) {
Jon Halldb2cf752016-06-28 16:24:45 -0700177 log.warn("Exception accessing metadata file at {}:", metadataUrl, e);
Madan Jampanif172d402016-03-04 00:56:38 -0800178 return false;
179 }
Madan Jampaniad3c5262016-01-20 00:50:17 -0800180 }
181
Jordan Haltermandbfff062017-06-19 10:31:32 -0700182 private Versioned<ClusterMetadata> blockForMetadata(String metadataUrl) {
Ray Milkey3717e602018-02-01 13:49:47 -0800183 long iterations = 0;
Jordan Haltermandbfff062017-06-19 10:31:32 -0700184 for (;;) {
Jordan Halterman9ada34a2017-10-23 16:31:50 -0700185 try {
186 Versioned<ClusterMetadata> metadata = fetchMetadata(metadataUrl);
187 if (metadata != null) {
188 return metadata;
189 }
190 } catch (Exception e) {
191 log.warn("Exception attempting to access metadata file at {}: {}", metadataUrl, e);
Jordan Haltermandbfff062017-06-19 10:31:32 -0700192 }
193
194 try {
Ray Milkey961b19f2018-02-05 13:47:55 -0800195 Thread.sleep((int) Math.pow(2, iterations < 7 ? ++iterations : iterations) * 10L);
Jordan Haltermandbfff062017-06-19 10:31:32 -0700196 } catch (InterruptedException e) {
Ray Milkey5c7d4882018-02-05 14:50:39 -0800197 Thread.currentThread().interrupt();
Ray Milkey6a51cb92018-03-06 09:03:03 -0800198 throw new IllegalStateException(e);
Jordan Haltermandbfff062017-06-19 10:31:32 -0700199 }
200 }
201 }
202
Jordan Haltermane458f002018-09-18 17:42:05 -0700203 private static NodeId getNodeId(NodePrototype node) {
204 if (node.getId() != null) {
205 return NodeId.nodeId(node.getId());
206 } else if (node.getHost() != null) {
207 return NodeId.nodeId(node.getHost());
208 } else if (node.getIp() != null) {
209 return NodeId.nodeId(node.getIp());
210 } else {
211 return NodeId.nodeId(UUID.randomUUID().toString());
212 }
213 }
214
215 private static String getNodeHost(NodePrototype node) {
216 if (node.getHost() != null) {
217 return node.getHost();
218 } else if (node.getIp() != null) {
219 return node.getIp();
220 } else {
221 throw new IllegalArgumentException(
222 "Malformed cluster configuration: No host specified for node " + node.getId());
223 }
224 }
225
226 private static int getNodePort(NodePrototype node) {
227 if (node.getPort() != null) {
228 return node.getPort();
229 }
230 return DefaultControllerNode.DEFAULT_PORT;
231 }
232
Madan Jampanif172d402016-03-04 00:56:38 -0800233 private Versioned<ClusterMetadata> fetchMetadata(String metadataUrl) {
Madan Jampaniad3c5262016-01-20 00:50:17 -0800234 try {
Madan Jampanif172d402016-03-04 00:56:38 -0800235 URL url = new URL(metadataUrl);
Jordan Halterman00e92da2018-05-22 23:05:52 -0700236 ClusterMetadataPrototype metadata = null;
Madan Jampanif172d402016-03-04 00:56:38 -0800237 long version = 0;
Jon Halla3fcf672017-03-28 16:53:22 -0700238 if ("file".equals(url.getProtocol())) {
Madan Jampanif172d402016-03-04 00:56:38 -0800239 File file = new File(metadataUrl.replaceFirst("file://", ""));
240 version = file.lastModified();
Jordan Halterman00e92da2018-05-22 23:05:52 -0700241 metadata = mapper.readValue(new FileInputStream(file), ClusterMetadataPrototype.class);
Jon Halla3fcf672017-03-28 16:53:22 -0700242 } else if ("http".equals(url.getProtocol())) {
Jordan Haltermanda268372017-09-19 11:18:08 -0700243 try {
244 HttpURLConnection conn = (HttpURLConnection) url.openConnection();
245 if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
246 log.warn("Could not reach metadata URL {}. Retrying...", url);
247 return null;
248 }
249 if (conn.getResponseCode() == HttpURLConnection.HTTP_NO_CONTENT) {
250 return null;
251 }
252 version = conn.getLastModified();
Jordan Halterman00e92da2018-05-22 23:05:52 -0700253 metadata = mapper.readValue(conn.getInputStream(), ClusterMetadataPrototype.class);
Jordan Haltermanda268372017-09-19 11:18:08 -0700254 } catch (IOException e) {
Jordan Halterman46c76902017-08-24 14:55:25 -0700255 log.warn("Could not reach metadata URL {}. Retrying...", url);
256 return null;
257 }
Madan Jampanif172d402016-03-04 00:56:38 -0800258 }
Jordan Haltermandbfff062017-06-19 10:31:32 -0700259
zhiyong ke60e40002017-04-13 13:50:47 +0800260 if (null == metadata) {
261 log.warn("Metadata is null in the function fetchMetadata");
262 throw new NullPointerException();
263 }
Jordan Haltermandbfff062017-06-19 10:31:32 -0700264
Jordan Halterman00e92da2018-05-22 23:05:52 -0700265 return new Versioned<>(new ClusterMetadata(
266 PROVIDER_ID,
267 metadata.getName(),
268 metadata.getNode() != null ?
269 new DefaultControllerNode(
Jordan Haltermane458f002018-09-18 17:42:05 -0700270 getNodeId(metadata.getNode()),
271 getNodeHost(metadata.getNode()),
272 getNodePort(metadata.getNode())) : null,
Jordan Halterman19c123a2018-07-30 13:57:19 -0700273 metadata.getController()
274 .stream()
Jordan Haltermane458f002018-09-18 17:42:05 -0700275 .map(node -> new DefaultControllerNode(getNodeId(node), getNodeHost(node), getNodePort(node)))
Jordan Halterman19c123a2018-07-30 13:57:19 -0700276 .collect(Collectors.toSet()),
277 metadata.getStorage()
278 .stream()
Jordan Haltermane458f002018-09-18 17:42:05 -0700279 .map(node -> new DefaultControllerNode(getNodeId(node), getNodeHost(node), getNodePort(node)))
Jordan Halterman19c123a2018-07-30 13:57:19 -0700280 .collect(Collectors.toSet())),
Jordan Halterman00e92da2018-05-22 23:05:52 -0700281 version);
Madan Jampaniad3c5262016-01-20 00:50:17 -0800282 } catch (IOException e) {
Ray Milkey6a51cb92018-03-06 09:03:03 -0800283 throw new IllegalArgumentException(e);
Madan Jampaniad3c5262016-01-20 00:50:17 -0800284 }
Madan Jampaniad3c5262016-01-20 00:50:17 -0800285 }
286
Madan Jampani898fcca2016-02-26 12:42:08 -0800287 /**
Madan Jampanif172d402016-03-04 00:56:38 -0800288 * Monitors the metadata url for any updates and notifies providerService accordingly.
Madan Jampani898fcca2016-02-26 12:42:08 -0800289 */
Madan Jampanif172d402016-03-04 00:56:38 -0800290 private void watchUrl(String metadataUrl) {
Ray Milkeye3708c72017-11-13 13:10:46 -0800291 if (!isAvailable()) {
292 return;
293 }
Madan Jampanif172d402016-03-04 00:56:38 -0800294 // TODO: We are merely polling the url.
295 // This can be easily addressed for files. For http urls we need to move to a push style protocol.
dvaddire49add802017-08-23 09:45:49 +0530296 try {
297 Versioned<ClusterMetadata> latestMetadata = fetchMetadata(metadataUrl);
298 if (cachedMetadata.get() != null && latestMetadata != null
299 && cachedMetadata.get().version() < latestMetadata.version()) {
300 cachedMetadata.set(latestMetadata);
301 providerService.clusterMetadataChanged(latestMetadata);
302 }
303 } catch (Exception e) {
304 log.error("Unable to parse metadata : ", e);
Madan Jampani898fcca2016-02-26 12:42:08 -0800305 }
306 }
Jordan Halterman00e92da2018-05-22 23:05:52 -0700307
308 private static class ClusterMetadataPrototype {
309 private String name;
310 private NodePrototype node;
Jordan Halterman19c123a2018-07-30 13:57:19 -0700311 private Set<NodePrototype> controller = Sets.newHashSet();
312 private Set<NodePrototype> storage = Sets.newHashSet();
Jordan Halterman00e92da2018-05-22 23:05:52 -0700313
314 public String getName() {
315 return name;
316 }
317
318 public void setName(String name) {
319 this.name = name;
320 }
321
322 public NodePrototype getNode() {
323 return node;
324 }
325
326 public void setNode(NodePrototype node) {
327 this.node = node;
328 }
329
Jordan Halterman19c123a2018-07-30 13:57:19 -0700330 public Set<NodePrototype> getController() {
331 return controller;
Jordan Halterman00e92da2018-05-22 23:05:52 -0700332 }
333
Jordan Halterman19c123a2018-07-30 13:57:19 -0700334 public void setController(Set<NodePrototype> controller) {
335 this.controller = controller;
336 }
337
338 public Set<NodePrototype> getStorage() {
339 return storage;
340 }
341
342 public void setStorage(Set<NodePrototype> storage) {
343 this.storage = storage;
Jordan Halterman00e92da2018-05-22 23:05:52 -0700344 }
345 }
346
347 private static class NodePrototype {
348 private String id;
349 private String ip;
Jordan Haltermane458f002018-09-18 17:42:05 -0700350 private String host;
Jordan Halterman00e92da2018-05-22 23:05:52 -0700351 private Integer port;
352
353 public String getId() {
354 return id;
355 }
356
357 public void setId(String id) {
358 this.id = id;
359 }
360
361 public String getIp() {
362 return ip;
363 }
364
365 public void setIp(String ip) {
366 this.ip = ip;
367 }
368
Jordan Haltermane458f002018-09-18 17:42:05 -0700369 public String getHost() {
370 return host;
371 }
372
373 public void setHost(String host) {
374 this.host = host;
375 }
376
Jordan Halterman00e92da2018-05-22 23:05:52 -0700377 public Integer getPort() {
378 return port;
379 }
380
381 public void setPort(Integer port) {
382 this.port = port;
383 }
384 }
Madan Jampaniad3c5262016-01-20 00:50:17 -0800385}