Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 1 | /* |
Brian O'Connor | a09fe5b | 2017-08-03 21:12:30 -0700 | [diff] [blame] | 2 | * Copyright 2016-present Open Networking Foundation |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package org.onosproject.store.primitives.impl; |
| 17 | |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 18 | import java.io.File; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 19 | import java.io.IOException; |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 20 | import java.nio.file.FileVisitResult; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 21 | import java.nio.file.Files; |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 22 | import java.nio.file.Path; |
| 23 | import java.nio.file.SimpleFileVisitor; |
| 24 | import java.nio.file.attribute.BasicFileAttributes; |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 25 | import java.time.Duration; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 26 | import java.util.Collection; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 27 | import java.util.concurrent.CompletableFuture; |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 28 | import java.util.stream.Collectors; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 29 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 30 | import io.atomix.protocols.raft.RaftServer; |
| 31 | import io.atomix.protocols.raft.cluster.MemberId; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 32 | import io.atomix.protocols.raft.storage.RaftStorage; |
| 33 | import io.atomix.storage.StorageLevel; |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 34 | import org.onosproject.cluster.Partition; |
Jordan Halterman | 28183ee | 2017-10-17 17:29:10 -0700 | [diff] [blame] | 35 | import org.onosproject.store.cluster.messaging.ClusterCommunicationService; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 36 | import org.onosproject.store.primitives.resources.impl.AtomixSerializerAdapter; |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 37 | import org.onosproject.store.service.PartitionInfo; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 38 | import org.onosproject.store.service.Serializer; |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 39 | import org.slf4j.Logger; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 40 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 41 | import static org.slf4j.LoggerFactory.getLogger; |
| 42 | |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 43 | /** |
| 44 | * {@link StoragePartition} server. |
| 45 | */ |
| 46 | public class StoragePartitionServer implements Managed<StoragePartitionServer> { |
| 47 | |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 48 | private final Logger log = getLogger(getClass()); |
| 49 | |
Jordan Halterman | 035231e | 2017-07-18 08:39:07 -0700 | [diff] [blame] | 50 | private static final int MAX_SEGMENT_SIZE = 1024 * 1024 * 64; |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 51 | private static final long ELECTION_TIMEOUT_MILLIS = 2500; |
Jordan Halterman | 19486e3 | 2017-11-02 15:00:06 -0700 | [diff] [blame] | 52 | private static final long HEARTBEAT_INTERVAL_MILLIS = 250; |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 53 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 54 | private final MemberId localMemberId; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 55 | private final StoragePartition partition; |
Jordan Halterman | 28183ee | 2017-10-17 17:29:10 -0700 | [diff] [blame] | 56 | private final ClusterCommunicationService clusterCommunicator; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 57 | private RaftServer server; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 58 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 59 | public StoragePartitionServer( |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 60 | StoragePartition partition, |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 61 | MemberId localMemberId, |
Jordan Halterman | 28183ee | 2017-10-17 17:29:10 -0700 | [diff] [blame] | 62 | ClusterCommunicationService clusterCommunicator) { |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 63 | this.partition = partition; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 64 | this.localMemberId = localMemberId; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 65 | this.clusterCommunicator = clusterCommunicator; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 66 | } |
| 67 | |
| 68 | @Override |
| 69 | public CompletableFuture<Void> open() { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 70 | log.info("Starting server for partition {} ({})", partition.getId(), partition.getVersion()); |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 71 | CompletableFuture<RaftServer> serverOpenFuture; |
| 72 | if (partition.getMemberIds().contains(localMemberId)) { |
Madan Jampani | 65f24bb | 2016-03-15 15:16:18 -0700 | [diff] [blame] | 73 | if (server != null && server.isRunning()) { |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 74 | return CompletableFuture.completedFuture(null); |
| 75 | } |
| 76 | synchronized (this) { |
Madan Jampani | 630e7ac | 2016-05-31 11:34:05 -0700 | [diff] [blame] | 77 | server = buildServer(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 78 | } |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 79 | serverOpenFuture = server.bootstrap(partition.getMemberIds()); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 80 | } else { |
| 81 | serverOpenFuture = CompletableFuture.completedFuture(null); |
| 82 | } |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 83 | return serverOpenFuture.whenComplete((r, e) -> { |
| 84 | if (e == null) { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 85 | log.info("Successfully started server for partition {} ({})", |
| 86 | partition.getId(), partition.getVersion()); |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 87 | } else { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 88 | log.info("Failed to start server for partition {} ({})", |
| 89 | partition.getId(), partition.getVersion(), e); |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 90 | } |
| 91 | }).thenApply(v -> null); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 92 | } |
| 93 | |
| 94 | @Override |
| 95 | public CompletableFuture<Void> close() { |
Madan Jampani | 630e7ac | 2016-05-31 11:34:05 -0700 | [diff] [blame] | 96 | return server.shutdown(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 97 | } |
| 98 | |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 99 | /** |
| 100 | * Closes the server and exits the partition. |
| 101 | * @return future that is completed when the operation is complete |
| 102 | */ |
| 103 | public CompletableFuture<Void> closeAndExit() { |
Madan Jampani | 630e7ac | 2016-05-31 11:34:05 -0700 | [diff] [blame] | 104 | return server.leave(); |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 105 | } |
| 106 | |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 107 | /** |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 108 | * Deletes the server. |
| 109 | */ |
| 110 | public void delete() { |
| 111 | try { |
| 112 | Files.walkFileTree(partition.getDataFolder().toPath(), new SimpleFileVisitor<Path>() { |
| 113 | @Override |
| 114 | public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { |
| 115 | Files.delete(file); |
| 116 | return FileVisitResult.CONTINUE; |
| 117 | } |
| 118 | @Override |
| 119 | public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { |
| 120 | Files.delete(dir); |
| 121 | return FileVisitResult.CONTINUE; |
| 122 | } |
| 123 | }); |
| 124 | } catch (IOException e) { |
| 125 | log.error("Failed to delete partition: {}", e); |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | /** |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 130 | * Forks the existing partition into a new partition. |
| 131 | * |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 132 | * @param fromPartition the partition from which to fork the server |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 133 | * @return future to be completed once the fork operation is complete |
| 134 | */ |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 135 | public CompletableFuture<Void> fork(Partition fromPartition) { |
| 136 | log.info("Forking server for partition {} ({}->{})", |
| 137 | partition.getId(), fromPartition.getVersion(), partition.getVersion()); |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 138 | RaftServer.Builder builder = RaftServer.newBuilder(localMemberId) |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 139 | .withName(String.format("partition-%s", fromPartition.getId())) |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 140 | .withProtocol(new RaftServerCommunicator( |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 141 | String.format("partition-%s-%s", fromPartition.getId(), fromPartition.getVersion()), |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 142 | Serializer.using(StorageNamespaces.RAFT_PROTOCOL), |
| 143 | clusterCommunicator)) |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 144 | .withElectionTimeout(Duration.ofMillis(ELECTION_TIMEOUT_MILLIS)) |
| 145 | .withHeartbeatInterval(Duration.ofMillis(HEARTBEAT_INTERVAL_MILLIS)) |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 146 | .withStorage(RaftStorage.newBuilder() |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 147 | .withPrefix(String.format("partition-%s", partition.getId())) |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 148 | .withStorageLevel(StorageLevel.MAPPED) |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 149 | .withSerializer(new AtomixSerializerAdapter(Serializer.using(StorageNamespaces.RAFT_STORAGE))) |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 150 | .withDirectory(partition.getDataFolder()) |
| 151 | .withMaxSegmentSize(MAX_SEGMENT_SIZE) |
| 152 | .build()); |
| 153 | StoragePartition.RAFT_SERVICES.forEach(builder::addService); |
| 154 | RaftServer server = builder.build(); |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 155 | |
| 156 | // Create a collection of members currently in the source partition. |
| 157 | Collection<MemberId> members = fromPartition.getMembers() |
| 158 | .stream() |
| 159 | .map(id -> MemberId.from(id.id())) |
| 160 | .collect(Collectors.toList()); |
| 161 | |
| 162 | // If this node is a member of the partition, join the partition. Otherwise, listen to the partition. |
| 163 | CompletableFuture<RaftServer> future = members.contains(localMemberId) |
| 164 | ? server.bootstrap(members) : server.listen(members); |
| 165 | |
| 166 | // TODO: We should leave the cluster for nodes that aren't normally members to ensure the source |
| 167 | // cluster's configuration is kept consistent for rolling back upgrades, but Atomix deletes configuration |
| 168 | // files when a node leaves the cluster so we can't do that here. |
| 169 | return future.thenCompose(v -> server.shutdown()) |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 170 | .thenCompose(v -> { |
| 171 | // Delete the cluster configuration file from the forked partition. |
| 172 | try { |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 173 | Files.delete(new File( |
| 174 | partition.getDataFolder(), |
| 175 | String.format("partition-%s.conf", partition.getId())).toPath()); |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 176 | } catch (IOException e) { |
| 177 | log.error("Failed to delete partition configuration: {}", e); |
| 178 | } |
| 179 | |
| 180 | // Build and bootstrap a new server. |
| 181 | this.server = buildServer(); |
| 182 | return this.server.bootstrap(); |
| 183 | }).whenComplete((r, e) -> { |
| 184 | if (e == null) { |
| 185 | log.info("Successfully forked server for partition {} ({}->{})", |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 186 | partition.getId(), fromPartition.getVersion(), partition.getVersion()); |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 187 | } else { |
| 188 | log.info("Failed to fork server for partition {} ({}->{})", |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 189 | partition.getId(), fromPartition.getVersion(), partition.getVersion(), e); |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 190 | } |
| 191 | }).thenApply(v -> null); |
| 192 | } |
| 193 | |
| 194 | private RaftServer buildServer() { |
| 195 | RaftServer.Builder builder = RaftServer.newBuilder(localMemberId) |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 196 | .withName(String.format("partition-%s", partition.getId())) |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 197 | .withProtocol(new RaftServerCommunicator( |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 198 | String.format("partition-%s-%s", partition.getId(), partition.getVersion()), |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 199 | Serializer.using(StorageNamespaces.RAFT_PROTOCOL), |
| 200 | clusterCommunicator)) |
| 201 | .withElectionTimeout(Duration.ofMillis(ELECTION_TIMEOUT_MILLIS)) |
| 202 | .withHeartbeatInterval(Duration.ofMillis(HEARTBEAT_INTERVAL_MILLIS)) |
| 203 | .withStorage(RaftStorage.newBuilder() |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 204 | .withPrefix(String.format("partition-%s", partition.getId())) |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame] | 205 | .withStorageLevel(StorageLevel.MAPPED) |
| 206 | .withSerializer(new AtomixSerializerAdapter(Serializer.using(StorageNamespaces.RAFT_STORAGE))) |
| 207 | .withDirectory(partition.getDataFolder()) |
Jordan Halterman | 035231e | 2017-07-18 08:39:07 -0700 | [diff] [blame] | 208 | .withMaxSegmentSize(MAX_SEGMENT_SIZE) |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 209 | .build()); |
| 210 | StoragePartition.RAFT_SERVICES.forEach(builder::addService); |
| 211 | return builder.build(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 212 | } |
| 213 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 214 | public CompletableFuture<Void> join(Collection<MemberId> otherMembers) { |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 215 | log.info("Joining partition {} ({})", partition.getId(), partition.getName()); |
Madan Jampani | 630e7ac | 2016-05-31 11:34:05 -0700 | [diff] [blame] | 216 | server = buildServer(); |
| 217 | return server.join(otherMembers).whenComplete((r, e) -> { |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 218 | if (e == null) { |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 219 | log.info("Successfully joined partition {} ({})", partition.getId(), partition.getName()); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 220 | } else { |
Jordan Halterman | 07f052b | 2017-10-08 14:22:41 -0700 | [diff] [blame] | 221 | log.info("Failed to join partition {} ({})", partition.getId(), partition.getName(), e); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 222 | } |
| 223 | }).thenApply(v -> null); |
| 224 | } |
| 225 | |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 226 | @Override |
| 227 | public boolean isOpen() { |
Madan Jampani | 65f24bb | 2016-03-15 15:16:18 -0700 | [diff] [blame] | 228 | return server.isRunning(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 229 | } |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 230 | |
| 231 | /** |
| 232 | * Returns the partition information. |
| 233 | * @return partition info |
| 234 | */ |
| 235 | public PartitionInfo info() { |
| 236 | return new StoragePartitionDetails(partition.getId(), |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 237 | server.cluster().getMembers(), |
| 238 | server.cluster().getMembers(), |
| 239 | server.cluster().getLeader(), |
| 240 | server.cluster().getTerm()).toPartitionInfo(); |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 241 | } |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 242 | } |