Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 1 | /* |
Brian O'Connor | a09fe5b | 2017-08-03 21:12:30 -0700 | [diff] [blame] | 2 | * Copyright 2016-present Open Networking Foundation |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package org.onosproject.cluster.impl; |
| 17 | |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 18 | import java.io.File; |
| 19 | import java.io.FileInputStream; |
| 20 | import java.io.IOException; |
| 21 | import java.net.HttpURLConnection; |
| 22 | import java.net.URL; |
| 23 | import java.util.Set; |
| 24 | import java.util.UUID; |
| 25 | import java.util.concurrent.ScheduledExecutorService; |
| 26 | import java.util.concurrent.TimeUnit; |
| 27 | import java.util.concurrent.atomic.AtomicReference; |
| 28 | import java.util.stream.Collectors; |
| 29 | |
Ray Milkey | 6a51cb9 | 2018-03-06 09:03:03 -0800 | [diff] [blame] | 30 | import com.fasterxml.jackson.databind.ObjectMapper; |
Ray Milkey | 6a51cb9 | 2018-03-06 09:03:03 -0800 | [diff] [blame] | 31 | import com.google.common.io.Files; |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 32 | import org.apache.felix.scr.annotations.Activate; |
| 33 | import org.apache.felix.scr.annotations.Component; |
| 34 | import org.apache.felix.scr.annotations.Deactivate; |
| 35 | import org.apache.felix.scr.annotations.Reference; |
| 36 | import org.apache.felix.scr.annotations.ReferenceCardinality; |
| 37 | import org.onlab.packet.IpAddress; |
| 38 | import org.onosproject.cluster.ClusterMetadata; |
| 39 | import org.onosproject.cluster.ClusterMetadataProvider; |
| 40 | import org.onosproject.cluster.ClusterMetadataProviderRegistry; |
| 41 | import org.onosproject.cluster.ClusterMetadataProviderService; |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 42 | import org.onosproject.cluster.DefaultControllerNode; |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 43 | import org.onosproject.cluster.Node; |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 44 | import org.onosproject.cluster.NodeId; |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 45 | import org.onosproject.cluster.PartitionId; |
| 46 | import org.onosproject.net.provider.ProviderId; |
| 47 | import org.onosproject.store.service.Versioned; |
| 48 | import org.slf4j.Logger; |
| 49 | |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 50 | import static com.google.common.base.Preconditions.checkState; |
Yuta HIGUCHI | 1624df1 | 2016-07-21 16:54:33 -0700 | [diff] [blame] | 51 | import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; |
Ray Milkey | 6a51cb9 | 2018-03-06 09:03:03 -0800 | [diff] [blame] | 52 | import static org.onlab.util.Tools.groupedThreads; |
| 53 | import static org.slf4j.LoggerFactory.getLogger; |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 54 | |
| 55 | /** |
| 56 | * Provider of {@link ClusterMetadata cluster metadata} sourced from a local config file. |
| 57 | */ |
| 58 | @Component(immediate = true) |
| 59 | public class ConfigFileBasedClusterMetadataProvider implements ClusterMetadataProvider { |
| 60 | |
| 61 | private final Logger log = getLogger(getClass()); |
| 62 | |
Madan Jampani | 898fcca | 2016-02-26 12:42:08 -0800 | [diff] [blame] | 63 | private static final String CONFIG_DIR = "../config"; |
| 64 | private static final String CONFIG_FILE_NAME = "cluster.json"; |
| 65 | private static final File CONFIG_FILE = new File(CONFIG_DIR, CONFIG_FILE_NAME); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 66 | |
| 67 | @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| 68 | protected ClusterMetadataProviderRegistry providerRegistry; |
| 69 | |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 70 | private static final ProviderId PROVIDER_ID = new ProviderId("file", "none"); |
Madan Jampani | 898fcca | 2016-02-26 12:42:08 -0800 | [diff] [blame] | 71 | private final AtomicReference<Versioned<ClusterMetadata>> cachedMetadata = new AtomicReference<>(); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 72 | private final ScheduledExecutorService configFileChangeDetector = |
Yuta HIGUCHI | 1624df1 | 2016-07-21 16:54:33 -0700 | [diff] [blame] | 73 | newSingleThreadScheduledExecutor(groupedThreads("onos/cluster/metadata/config-watcher", "", log)); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 74 | |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 75 | private String metadataUrl; |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 76 | private ObjectMapper mapper; |
| 77 | private ClusterMetadataProviderService providerService; |
| 78 | |
| 79 | @Activate |
| 80 | public void activate() { |
| 81 | mapper = new ObjectMapper(); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 82 | providerService = providerRegistry.register(this); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 83 | metadataUrl = System.getProperty("onos.cluster.metadata.uri", "file://" + CONFIG_DIR + "/" + CONFIG_FILE); |
| 84 | configFileChangeDetector.scheduleWithFixedDelay(() -> watchUrl(metadataUrl), 100, 500, TimeUnit.MILLISECONDS); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 85 | log.info("Started"); |
| 86 | } |
| 87 | |
| 88 | @Deactivate |
| 89 | public void deactivate() { |
Madan Jampani | 898fcca | 2016-02-26 12:42:08 -0800 | [diff] [blame] | 90 | configFileChangeDetector.shutdown(); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 91 | providerRegistry.unregister(this); |
| 92 | log.info("Stopped"); |
| 93 | } |
| 94 | |
| 95 | @Override |
| 96 | public ProviderId id() { |
| 97 | return PROVIDER_ID; |
| 98 | } |
| 99 | |
| 100 | @Override |
| 101 | public Versioned<ClusterMetadata> getClusterMetadata() { |
| 102 | checkState(isAvailable()); |
| 103 | synchronized (this) { |
| 104 | if (cachedMetadata.get() == null) { |
Jordan Halterman | dbfff06 | 2017-06-19 10:31:32 -0700 | [diff] [blame] | 105 | cachedMetadata.set(blockForMetadata(metadataUrl)); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 106 | } |
| 107 | return cachedMetadata.get(); |
| 108 | } |
| 109 | } |
| 110 | |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 111 | private ClusterMetadataPrototype toPrototype(ClusterMetadata metadata) { |
| 112 | ClusterMetadataPrototype prototype = new ClusterMetadataPrototype(); |
| 113 | prototype.setName(metadata.getName()); |
| 114 | prototype.setCluster(metadata.getNodes() |
| 115 | .stream() |
| 116 | .map(this::toPrototype) |
| 117 | .collect(Collectors.toSet())); |
| 118 | return prototype; |
| 119 | } |
| 120 | |
| 121 | private NodePrototype toPrototype(Node node) { |
| 122 | NodePrototype prototype = new NodePrototype(); |
| 123 | prototype.setId(node.id().id()); |
| 124 | prototype.setIp(node.ip().toString()); |
| 125 | prototype.setPort(node.tcpPort()); |
| 126 | return prototype; |
| 127 | } |
| 128 | |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 129 | @Override |
| 130 | public void setClusterMetadata(ClusterMetadata metadata) { |
| 131 | try { |
Ray Milkey | 8320088 | 2017-11-13 19:18:21 -0800 | [diff] [blame] | 132 | File configFile = new File(metadataUrl.replaceFirst("file://", "")); |
| 133 | Files.createParentDirs(configFile); |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 134 | ClusterMetadataPrototype metadataPrototype = toPrototype(metadata); |
| 135 | mapper.writeValue(configFile, metadataPrototype); |
Ray Milkey | 8320088 | 2017-11-13 19:18:21 -0800 | [diff] [blame] | 136 | cachedMetadata.set(fetchMetadata(metadataUrl)); |
| 137 | providerService.clusterMetadataChanged(new Versioned<>(metadata, configFile.lastModified())); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 138 | } catch (IOException e) { |
Ray Milkey | 6a51cb9 | 2018-03-06 09:03:03 -0800 | [diff] [blame] | 139 | throw new IllegalArgumentException(e); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 140 | } |
| 141 | } |
| 142 | |
| 143 | @Override |
| 144 | public void addActivePartitionMember(PartitionId partitionId, NodeId nodeId) { |
| 145 | throw new UnsupportedOperationException(); |
| 146 | } |
| 147 | |
| 148 | @Override |
| 149 | public void removeActivePartitionMember(PartitionId partitionId, NodeId nodeId) { |
| 150 | throw new UnsupportedOperationException(); |
| 151 | } |
| 152 | |
| 153 | @Override |
| 154 | public Set<NodeId> getActivePartitionMembers(PartitionId partitionId) { |
| 155 | throw new UnsupportedOperationException(); |
| 156 | } |
| 157 | |
| 158 | @Override |
| 159 | public boolean isAvailable() { |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 160 | try { |
| 161 | URL url = new URL(metadataUrl); |
Jon Hall | a3fcf67 | 2017-03-28 16:53:22 -0700 | [diff] [blame] | 162 | if ("file".equals(url.getProtocol())) { |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 163 | File file = new File(metadataUrl.replaceFirst("file://", "")); |
| 164 | return file.exists(); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 165 | } else { |
Jordan Halterman | 46c7690 | 2017-08-24 14:55:25 -0700 | [diff] [blame] | 166 | // Return true for HTTP URLs since we allow blocking until HTTP servers come up |
| 167 | return "http".equals(url.getProtocol()); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 168 | } |
| 169 | } catch (Exception e) { |
Jon Hall | db2cf75 | 2016-06-28 16:24:45 -0700 | [diff] [blame] | 170 | log.warn("Exception accessing metadata file at {}:", metadataUrl, e); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 171 | return false; |
| 172 | } |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 173 | } |
| 174 | |
Jordan Halterman | dbfff06 | 2017-06-19 10:31:32 -0700 | [diff] [blame] | 175 | private Versioned<ClusterMetadata> blockForMetadata(String metadataUrl) { |
Ray Milkey | 3717e60 | 2018-02-01 13:49:47 -0800 | [diff] [blame] | 176 | long iterations = 0; |
Jordan Halterman | dbfff06 | 2017-06-19 10:31:32 -0700 | [diff] [blame] | 177 | for (;;) { |
Jordan Halterman | 9ada34a | 2017-10-23 16:31:50 -0700 | [diff] [blame] | 178 | try { |
| 179 | Versioned<ClusterMetadata> metadata = fetchMetadata(metadataUrl); |
| 180 | if (metadata != null) { |
| 181 | return metadata; |
| 182 | } |
| 183 | } catch (Exception e) { |
| 184 | log.warn("Exception attempting to access metadata file at {}: {}", metadataUrl, e); |
Jordan Halterman | dbfff06 | 2017-06-19 10:31:32 -0700 | [diff] [blame] | 185 | } |
| 186 | |
| 187 | try { |
Ray Milkey | 961b19f | 2018-02-05 13:47:55 -0800 | [diff] [blame] | 188 | Thread.sleep((int) Math.pow(2, iterations < 7 ? ++iterations : iterations) * 10L); |
Jordan Halterman | dbfff06 | 2017-06-19 10:31:32 -0700 | [diff] [blame] | 189 | } catch (InterruptedException e) { |
Ray Milkey | 5c7d488 | 2018-02-05 14:50:39 -0800 | [diff] [blame] | 190 | Thread.currentThread().interrupt(); |
Ray Milkey | 6a51cb9 | 2018-03-06 09:03:03 -0800 | [diff] [blame] | 191 | throw new IllegalStateException(e); |
Jordan Halterman | dbfff06 | 2017-06-19 10:31:32 -0700 | [diff] [blame] | 192 | } |
| 193 | } |
| 194 | } |
| 195 | |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 196 | private Versioned<ClusterMetadata> fetchMetadata(String metadataUrl) { |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 197 | try { |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 198 | URL url = new URL(metadataUrl); |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 199 | ClusterMetadataPrototype metadata = null; |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 200 | long version = 0; |
Jon Hall | a3fcf67 | 2017-03-28 16:53:22 -0700 | [diff] [blame] | 201 | if ("file".equals(url.getProtocol())) { |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 202 | File file = new File(metadataUrl.replaceFirst("file://", "")); |
| 203 | version = file.lastModified(); |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 204 | metadata = mapper.readValue(new FileInputStream(file), ClusterMetadataPrototype.class); |
Jon Hall | a3fcf67 | 2017-03-28 16:53:22 -0700 | [diff] [blame] | 205 | } else if ("http".equals(url.getProtocol())) { |
Jordan Halterman | da26837 | 2017-09-19 11:18:08 -0700 | [diff] [blame] | 206 | try { |
| 207 | HttpURLConnection conn = (HttpURLConnection) url.openConnection(); |
| 208 | if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { |
| 209 | log.warn("Could not reach metadata URL {}. Retrying...", url); |
| 210 | return null; |
| 211 | } |
| 212 | if (conn.getResponseCode() == HttpURLConnection.HTTP_NO_CONTENT) { |
| 213 | return null; |
| 214 | } |
| 215 | version = conn.getLastModified(); |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 216 | metadata = mapper.readValue(conn.getInputStream(), ClusterMetadataPrototype.class); |
Jordan Halterman | da26837 | 2017-09-19 11:18:08 -0700 | [diff] [blame] | 217 | } catch (IOException e) { |
Jordan Halterman | 46c7690 | 2017-08-24 14:55:25 -0700 | [diff] [blame] | 218 | log.warn("Could not reach metadata URL {}. Retrying...", url); |
| 219 | return null; |
| 220 | } |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 221 | } |
Jordan Halterman | dbfff06 | 2017-06-19 10:31:32 -0700 | [diff] [blame] | 222 | |
zhiyong ke | 60e4000 | 2017-04-13 13:50:47 +0800 | [diff] [blame] | 223 | if (null == metadata) { |
| 224 | log.warn("Metadata is null in the function fetchMetadata"); |
| 225 | throw new NullPointerException(); |
| 226 | } |
Jordan Halterman | dbfff06 | 2017-06-19 10:31:32 -0700 | [diff] [blame] | 227 | |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 228 | return new Versioned<>(new ClusterMetadata( |
| 229 | PROVIDER_ID, |
| 230 | metadata.getName(), |
| 231 | metadata.getNode() != null ? |
| 232 | new DefaultControllerNode( |
| 233 | metadata.getNode().getId() != null |
| 234 | ? NodeId.nodeId(metadata.getNode().getId()) |
| 235 | : metadata.getNode().getIp() != null |
| 236 | ? NodeId.nodeId(IpAddress.valueOf(metadata.getNode().getIp()).toString()) |
| 237 | : NodeId.nodeId(UUID.randomUUID().toString()), |
| 238 | metadata.getNode().getIp() != null |
| 239 | ? IpAddress.valueOf(metadata.getNode().getIp()) |
| 240 | : null, |
| 241 | metadata.getNode().getPort() != null |
| 242 | ? metadata.getNode().getPort() |
| 243 | : DefaultControllerNode.DEFAULT_PORT) : null, |
| 244 | metadata.getCluster() |
| 245 | .stream() |
| 246 | .map(node -> new DefaultControllerNode( |
| 247 | NodeId.nodeId(node.getId()), |
| 248 | IpAddress.valueOf(node.getIp()), |
| 249 | node.getPort() != null ? node.getPort() : 5679)) |
| 250 | .collect(Collectors.toSet())), |
| 251 | version); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 252 | } catch (IOException e) { |
Ray Milkey | 6a51cb9 | 2018-03-06 09:03:03 -0800 | [diff] [blame] | 253 | throw new IllegalArgumentException(e); |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 254 | } |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 255 | } |
| 256 | |
Madan Jampani | 898fcca | 2016-02-26 12:42:08 -0800 | [diff] [blame] | 257 | /** |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 258 | * Monitors the metadata url for any updates and notifies providerService accordingly. |
Madan Jampani | 898fcca | 2016-02-26 12:42:08 -0800 | [diff] [blame] | 259 | */ |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 260 | private void watchUrl(String metadataUrl) { |
Ray Milkey | e3708c7 | 2017-11-13 13:10:46 -0800 | [diff] [blame] | 261 | if (!isAvailable()) { |
| 262 | return; |
| 263 | } |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 264 | // TODO: We are merely polling the url. |
| 265 | // This can be easily addressed for files. For http urls we need to move to a push style protocol. |
dvaddire | 49add80 | 2017-08-23 09:45:49 +0530 | [diff] [blame] | 266 | try { |
| 267 | Versioned<ClusterMetadata> latestMetadata = fetchMetadata(metadataUrl); |
| 268 | if (cachedMetadata.get() != null && latestMetadata != null |
| 269 | && cachedMetadata.get().version() < latestMetadata.version()) { |
| 270 | cachedMetadata.set(latestMetadata); |
| 271 | providerService.clusterMetadataChanged(latestMetadata); |
| 272 | } |
| 273 | } catch (Exception e) { |
| 274 | log.error("Unable to parse metadata : ", e); |
Madan Jampani | 898fcca | 2016-02-26 12:42:08 -0800 | [diff] [blame] | 275 | } |
| 276 | } |
Jordan Halterman | 00e92da | 2018-05-22 23:05:52 -0700 | [diff] [blame^] | 277 | |
| 278 | private static class ClusterMetadataPrototype { |
| 279 | private String name; |
| 280 | private NodePrototype node; |
| 281 | private Set<NodePrototype> cluster; |
| 282 | |
| 283 | public String getName() { |
| 284 | return name; |
| 285 | } |
| 286 | |
| 287 | public void setName(String name) { |
| 288 | this.name = name; |
| 289 | } |
| 290 | |
| 291 | public NodePrototype getNode() { |
| 292 | return node; |
| 293 | } |
| 294 | |
| 295 | public void setNode(NodePrototype node) { |
| 296 | this.node = node; |
| 297 | } |
| 298 | |
| 299 | public Set<NodePrototype> getCluster() { |
| 300 | return cluster; |
| 301 | } |
| 302 | |
| 303 | public void setCluster(Set<NodePrototype> cluster) { |
| 304 | this.cluster = cluster; |
| 305 | } |
| 306 | } |
| 307 | |
| 308 | private static class NodePrototype { |
| 309 | private String id; |
| 310 | private String ip; |
| 311 | private Integer port; |
| 312 | |
| 313 | public String getId() { |
| 314 | return id; |
| 315 | } |
| 316 | |
| 317 | public void setId(String id) { |
| 318 | this.id = id; |
| 319 | } |
| 320 | |
| 321 | public String getIp() { |
| 322 | return ip; |
| 323 | } |
| 324 | |
| 325 | public void setIp(String ip) { |
| 326 | this.ip = ip; |
| 327 | } |
| 328 | |
| 329 | public Integer getPort() { |
| 330 | return port; |
| 331 | } |
| 332 | |
| 333 | public void setPort(Integer port) { |
| 334 | this.port = port; |
| 335 | } |
| 336 | } |
Madan Jampani | ad3c526 | 2016-01-20 00:50:17 -0800 | [diff] [blame] | 337 | } |