Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 1 | /* |
Brian O'Connor | a09fe5b | 2017-08-03 21:12:30 -0700 | [diff] [blame] | 2 | * Copyright 2016-present Open Networking Foundation |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package org.onosproject.store.primitives.impl; |
| 17 | |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 18 | import java.io.File; |
| 19 | import java.util.Collection; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 20 | import java.util.Map; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 21 | import java.util.Optional; |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 22 | import java.util.Set; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 23 | import java.util.concurrent.CompletableFuture; |
| 24 | import java.util.concurrent.atomic.AtomicBoolean; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 25 | import java.util.function.Supplier; |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 26 | import java.util.stream.Collectors; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 27 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 28 | import com.google.common.collect.Collections2; |
| 29 | import com.google.common.collect.ImmutableMap; |
| 30 | import io.atomix.protocols.raft.cluster.MemberId; |
| 31 | import io.atomix.protocols.raft.service.RaftService; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 32 | import org.onosproject.cluster.MembershipService; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 33 | import org.onosproject.cluster.NodeId; |
| 34 | import org.onosproject.cluster.Partition; |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 35 | import org.onosproject.cluster.PartitionId; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 36 | import org.onosproject.core.Version; |
| 37 | import org.onosproject.store.cluster.messaging.UnifiedClusterCommunicationService; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 38 | import org.onosproject.store.primitives.resources.impl.AtomixAtomicCounterMapService; |
| 39 | import org.onosproject.store.primitives.resources.impl.AtomixConsistentMapService; |
| 40 | import org.onosproject.store.primitives.resources.impl.AtomixConsistentSetMultimapService; |
| 41 | import org.onosproject.store.primitives.resources.impl.AtomixConsistentTreeMapService; |
| 42 | import org.onosproject.store.primitives.resources.impl.AtomixCounterService; |
| 43 | import org.onosproject.store.primitives.resources.impl.AtomixDocumentTreeService; |
| 44 | import org.onosproject.store.primitives.resources.impl.AtomixLeaderElectorService; |
| 45 | import org.onosproject.store.primitives.resources.impl.AtomixWorkQueueService; |
| 46 | import org.onosproject.store.service.DistributedPrimitive; |
Jordan Halterman | d0d8035 | 2017-08-10 15:08:27 -0700 | [diff] [blame] | 47 | import org.onosproject.store.service.Ordering; |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 48 | import org.onosproject.store.service.PartitionInfo; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 49 | import org.onosproject.store.service.Serializer; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 50 | |
| 51 | /** |
| 52 | * Storage partition. |
| 53 | */ |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 54 | public class StoragePartition implements Managed<StoragePartition> { |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 55 | |
| 56 | private final AtomicBoolean isOpened = new AtomicBoolean(false); |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 57 | private final UnifiedClusterCommunicationService clusterCommunicator; |
| 58 | private final MembershipService clusterService; |
| 59 | private final Version version; |
| 60 | private final Version source; |
| 61 | private final File dataFolder; |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 62 | private Partition partition; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 63 | private NodeId localNodeId; |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 64 | private StoragePartitionServer server; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 65 | private StoragePartitionClient client; |
| 66 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 67 | public static final Map<String, Supplier<RaftService>> RAFT_SERVICES = |
| 68 | ImmutableMap.<String, Supplier<RaftService>>builder() |
| 69 | .put(DistributedPrimitive.Type.CONSISTENT_MAP.name(), AtomixConsistentMapService::new) |
| 70 | .put(DistributedPrimitive.Type.CONSISTENT_TREEMAP.name(), AtomixConsistentTreeMapService::new) |
| 71 | .put(DistributedPrimitive.Type.CONSISTENT_MULTIMAP.name(), AtomixConsistentSetMultimapService::new) |
| 72 | .put(DistributedPrimitive.Type.COUNTER_MAP.name(), AtomixAtomicCounterMapService::new) |
| 73 | .put(DistributedPrimitive.Type.COUNTER.name(), AtomixCounterService::new) |
| 74 | .put(DistributedPrimitive.Type.LEADER_ELECTOR.name(), AtomixLeaderElectorService::new) |
| 75 | .put(DistributedPrimitive.Type.WORK_QUEUE.name(), AtomixWorkQueueService::new) |
Jordan Halterman | d0d8035 | 2017-08-10 15:08:27 -0700 | [diff] [blame] | 76 | .put(DistributedPrimitive.Type.DOCUMENT_TREE.name(), |
| 77 | () -> new AtomixDocumentTreeService(Ordering.NATURAL)) |
| 78 | .put(String.format("%s-%s", DistributedPrimitive.Type.DOCUMENT_TREE.name(), Ordering.NATURAL), |
| 79 | () -> new AtomixDocumentTreeService(Ordering.NATURAL)) |
| 80 | .put(String.format("%s-%s", DistributedPrimitive.Type.DOCUMENT_TREE.name(), Ordering.INSERTION), |
| 81 | () -> new AtomixDocumentTreeService(Ordering.INSERTION)) |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 82 | .build(); |
Madan Jampani | 65f24bb | 2016-03-15 15:16:18 -0700 | [diff] [blame] | 83 | |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 84 | public StoragePartition( |
| 85 | Partition partition, |
| 86 | Version version, |
| 87 | Version source, |
| 88 | UnifiedClusterCommunicationService clusterCommunicator, |
| 89 | MembershipService clusterService, |
| 90 | File dataFolder) { |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 91 | this.partition = partition; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 92 | this.version = version; |
| 93 | this.source = source; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 94 | this.clusterCommunicator = clusterCommunicator; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 95 | this.clusterService = clusterService; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 96 | this.localNodeId = clusterService.getLocalNode().id(); |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 97 | this.dataFolder = dataFolder; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 98 | } |
| 99 | |
Madan Jampani | 3a9911c | 2016-02-21 11:25:45 -0800 | [diff] [blame] | 100 | /** |
| 101 | * Returns the partition client instance. |
| 102 | * @return client |
| 103 | */ |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 104 | public StoragePartitionClient client() { |
| 105 | return client; |
| 106 | } |
| 107 | |
| 108 | @Override |
| 109 | public CompletableFuture<Void> open() { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 110 | if (source != null) { |
| 111 | return forkServer(source) |
| 112 | .thenCompose(v -> openClient()) |
| 113 | .thenAccept(v -> isOpened.set(true)) |
| 114 | .thenApply(v -> null); |
| 115 | } else if (partition.getMembers().contains(localNodeId)) { |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 116 | return openServer() |
| 117 | .thenCompose(v -> openClient()) |
| 118 | .thenAccept(v -> isOpened.set(true)) |
| 119 | .thenApply(v -> null); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 120 | } |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 121 | return openClient() |
| 122 | .thenAccept(v -> isOpened.set(true)) |
| 123 | .thenApply(v -> null); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 124 | } |
| 125 | |
| 126 | @Override |
| 127 | public CompletableFuture<Void> close() { |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 128 | // We do not explicitly close the server and instead let the cluster |
| 129 | // deal with this as an unclean exit. |
Madan Jampani | 65f24bb | 2016-03-15 15:16:18 -0700 | [diff] [blame] | 130 | return closeClient(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 131 | } |
| 132 | |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 133 | /** |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 134 | * Returns the partition name. |
| 135 | * |
| 136 | * @return the partition name |
| 137 | */ |
| 138 | public String getName() { |
| 139 | return getName(version); |
| 140 | } |
| 141 | |
| 142 | /** |
| 143 | * Returns the partition name for the given version. |
| 144 | * |
| 145 | * @param version the version for which to return the partition name |
| 146 | * @return the partition name for the given version |
| 147 | */ |
| 148 | String getName(Version version) { |
| 149 | return version != null ? String.format("partition-%d-%s", partition.getId().id(), version) : "partition-core"; |
| 150 | } |
| 151 | |
| 152 | /** |
| 153 | * Returns the partition version. |
| 154 | * |
| 155 | * @return the partition version |
| 156 | */ |
| 157 | public Version getVersion() { |
| 158 | return version; |
| 159 | } |
| 160 | |
| 161 | /** |
| 162 | * Returns the partition data folder. |
| 163 | * |
| 164 | * @return the partition data folder |
| 165 | */ |
| 166 | public File getDataFolder() { |
| 167 | return dataFolder; |
| 168 | } |
| 169 | |
| 170 | /** |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 171 | * Returns the identifier of the {@link Partition partition} associated with this instance. |
| 172 | * @return partition identifier |
| 173 | */ |
| 174 | public PartitionId getId() { |
| 175 | return partition.getId(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 176 | } |
| 177 | |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 178 | /** |
| 179 | * Returns the identifiers of partition members. |
| 180 | * @return partition member instance ids |
| 181 | */ |
| 182 | public Collection<NodeId> getMembers() { |
| 183 | return partition.getMembers(); |
| 184 | } |
| 185 | |
| 186 | /** |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 187 | * Returns the {@link MemberId identifiers} of partition members. |
| 188 | * @return partition member identifiers |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 189 | */ |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 190 | public Collection<MemberId> getMemberIds() { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 191 | return source != null ? |
| 192 | clusterService.getNodes() |
| 193 | .stream() |
| 194 | .map(node -> MemberId.from(node.id().id())) |
| 195 | .collect(Collectors.toList()) : |
| 196 | Collections2.transform(partition.getMembers(), n -> MemberId.from(n.id())); |
| 197 | } |
| 198 | |
| 199 | Collection<MemberId> getMemberIds(Version version) { |
| 200 | if (source == null || version.equals(source)) { |
| 201 | return Collections2.transform(partition.getMembers(), n -> MemberId.from(n.id())); |
| 202 | } else { |
| 203 | return clusterService.getNodes() |
| 204 | .stream() |
| 205 | .map(node -> MemberId.from(node.id().id())) |
| 206 | .collect(Collectors.toList()); |
| 207 | } |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 208 | } |
| 209 | |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 210 | /** |
| 211 | * Attempts to rejoin the partition. |
| 212 | * @return future that is completed after the operation is complete |
| 213 | */ |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 214 | private CompletableFuture<Void> openServer() { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 215 | StoragePartitionServer server = new StoragePartitionServer( |
| 216 | this, |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 217 | MemberId.from(localNodeId.id()), |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 218 | clusterCommunicator); |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 219 | return server.open().thenRun(() -> this.server = server); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 220 | } |
| 221 | |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 222 | /** |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 223 | * Forks the server from the given version. |
| 224 | * |
| 225 | * @return future to be completed once the server has been forked |
| 226 | */ |
| 227 | private CompletableFuture<Void> forkServer(Version version) { |
| 228 | StoragePartitionServer server = new StoragePartitionServer( |
| 229 | this, |
| 230 | MemberId.from(localNodeId.id()), |
| 231 | clusterCommunicator); |
| 232 | |
| 233 | CompletableFuture<Void> future; |
| 234 | if (clusterService.getNodes().size() == 1) { |
| 235 | future = server.fork(version); |
| 236 | } else { |
| 237 | future = server.join(clusterService.getNodes().stream() |
| 238 | .filter(node -> !node.id().equals(localNodeId)) |
| 239 | .map(node -> MemberId.from(node.id().id())) |
| 240 | .collect(Collectors.toList())); |
| 241 | } |
| 242 | return future.thenRun(() -> this.server = server); |
| 243 | } |
| 244 | |
| 245 | /** |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 246 | * Attempts to join the partition as a new member. |
| 247 | * @return future that is completed after the operation is complete |
| 248 | */ |
| 249 | private CompletableFuture<Void> joinCluster() { |
| 250 | Set<NodeId> otherMembers = partition.getMembers() |
| 251 | .stream() |
| 252 | .filter(nodeId -> !nodeId.equals(localNodeId)) |
| 253 | .collect(Collectors.toSet()); |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 254 | StoragePartitionServer server = new StoragePartitionServer(this, |
| 255 | MemberId.from(localNodeId.id()), |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 256 | clusterCommunicator); |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 257 | return server.join(Collections2.transform(otherMembers, n -> MemberId.from(n.id()))) |
| 258 | .thenRun(() -> this.server = server); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 259 | } |
| 260 | |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 261 | private CompletableFuture<StoragePartitionClient> openClient() { |
| 262 | client = new StoragePartitionClient(this, |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 263 | MemberId.from(localNodeId.id()), |
| 264 | new RaftClientCommunicator( |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 265 | getName(), |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 266 | Serializer.using(StorageNamespaces.RAFT_PROTOCOL), |
| 267 | clusterCommunicator)); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 268 | return client.open().thenApply(v -> client); |
| 269 | } |
| 270 | |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 271 | /** |
| 272 | * Closes the partition server if it was previously opened. |
| 273 | * @return future that is completed when the operation completes |
| 274 | */ |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 275 | public CompletableFuture<Void> leaveCluster() { |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 276 | return server != null ? server.closeAndExit() : CompletableFuture.completedFuture(null); |
| 277 | } |
| 278 | |
| 279 | @Override |
| 280 | public boolean isOpen() { |
Madan Jampani | 65f24bb | 2016-03-15 15:16:18 -0700 | [diff] [blame] | 281 | return isOpened.get(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 282 | } |
| 283 | |
| 284 | private CompletableFuture<Void> closeClient() { |
| 285 | if (client != null) { |
| 286 | return client.close(); |
| 287 | } |
| 288 | return CompletableFuture.completedFuture(null); |
| 289 | } |
| 290 | |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 291 | /** |
| 292 | * Returns the partition information if this partition is locally managed i.e. |
| 293 | * this node is a active member of the partition. |
| 294 | * @return partition info |
| 295 | */ |
| 296 | public Optional<PartitionInfo> info() { |
Madan Jampani | 65f24bb | 2016-03-15 15:16:18 -0700 | [diff] [blame] | 297 | return server != null && server.isOpen() ? Optional.of(server.info()) : Optional.empty(); |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 298 | } |
| 299 | |
Jon Hall | 1195afb | 2016-06-28 18:54:07 -0700 | [diff] [blame] | 300 | /** |
| 301 | * Process updates to partitions and handles joining or leaving a partition. |
| 302 | * @param newValue new Partition |
| 303 | */ |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 304 | public void onUpdate(Partition newValue) { |
Jon Hall | 1195afb | 2016-06-28 18:54:07 -0700 | [diff] [blame] | 305 | |
| 306 | boolean wasPresent = partition.getMembers().contains(localNodeId); |
| 307 | boolean isPresent = newValue.getMembers().contains(localNodeId); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 308 | this.partition = newValue; |
Jon Hall | 1195afb | 2016-06-28 18:54:07 -0700 | [diff] [blame] | 309 | if ((wasPresent && isPresent) || (!wasPresent && !isPresent)) { |
| 310 | // no action needed |
| 311 | return; |
| 312 | } |
| 313 | //only need to do action if our membership changed |
| 314 | if (wasPresent) { |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 315 | leaveCluster(); |
Jon Hall | 1195afb | 2016-06-28 18:54:07 -0700 | [diff] [blame] | 316 | } else if (isPresent) { |
| 317 | joinCluster(); |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 318 | } |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 319 | } |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 320 | } |