Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 1 | /* |
Brian O'Connor | a09fe5b | 2017-08-03 21:12:30 -0700 | [diff] [blame] | 2 | * Copyright 2016-present Open Networking Foundation |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package org.onosproject.store.primitives.impl; |
| 17 | |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 18 | import java.io.File; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 19 | import java.io.IOException; |
| 20 | import java.nio.file.Files; |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 21 | import java.time.Duration; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 22 | import java.util.Collection; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 23 | import java.util.concurrent.CompletableFuture; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 24 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 25 | import io.atomix.protocols.raft.RaftServer; |
| 26 | import io.atomix.protocols.raft.cluster.MemberId; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 27 | import io.atomix.protocols.raft.cluster.RaftMember; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 28 | import io.atomix.protocols.raft.storage.RaftStorage; |
| 29 | import io.atomix.storage.StorageLevel; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 30 | import org.onosproject.core.Version; |
| 31 | import org.onosproject.store.cluster.messaging.UnifiedClusterCommunicationService; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 32 | import org.onosproject.store.primitives.resources.impl.AtomixSerializerAdapter; |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 33 | import org.onosproject.store.service.PartitionInfo; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 34 | import org.onosproject.store.service.Serializer; |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 35 | import org.slf4j.Logger; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 36 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 37 | import static org.slf4j.LoggerFactory.getLogger; |
| 38 | |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 39 | /** |
| 40 | * {@link StoragePartition} server. |
| 41 | */ |
| 42 | public class StoragePartitionServer implements Managed<StoragePartitionServer> { |
| 43 | |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 44 | private final Logger log = getLogger(getClass()); |
| 45 | |
Jordan Halterman | 035231e | 2017-07-18 08:39:07 -0700 | [diff] [blame] | 46 | private static final int MAX_SEGMENT_SIZE = 1024 * 1024 * 64; |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 47 | private static final long ELECTION_TIMEOUT_MILLIS = 2500; |
| 48 | private static final long HEARTBEAT_INTERVAL_MILLIS = 1000; |
| 49 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 50 | private final MemberId localMemberId; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 51 | private final StoragePartition partition; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 52 | private final UnifiedClusterCommunicationService clusterCommunicator; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 53 | private RaftServer server; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 54 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 55 | public StoragePartitionServer( |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 56 | StoragePartition partition, |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 57 | MemberId localMemberId, |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 58 | UnifiedClusterCommunicationService clusterCommunicator) { |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 59 | this.partition = partition; |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 60 | this.localMemberId = localMemberId; |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 61 | this.clusterCommunicator = clusterCommunicator; |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 62 | } |
| 63 | |
| 64 | @Override |
| 65 | public CompletableFuture<Void> open() { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 66 | log.info("Starting server for partition {} ({})", partition.getId(), partition.getVersion()); |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 67 | CompletableFuture<RaftServer> serverOpenFuture; |
| 68 | if (partition.getMemberIds().contains(localMemberId)) { |
Madan Jampani | 65f24bb | 2016-03-15 15:16:18 -0700 | [diff] [blame] | 69 | if (server != null && server.isRunning()) { |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 70 | return CompletableFuture.completedFuture(null); |
| 71 | } |
| 72 | synchronized (this) { |
Madan Jampani | 630e7ac | 2016-05-31 11:34:05 -0700 | [diff] [blame] | 73 | server = buildServer(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 74 | } |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 75 | serverOpenFuture = server.bootstrap(partition.getMemberIds()); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 76 | } else { |
| 77 | serverOpenFuture = CompletableFuture.completedFuture(null); |
| 78 | } |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 79 | return serverOpenFuture.whenComplete((r, e) -> { |
| 80 | if (e == null) { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 81 | log.info("Successfully started server for partition {} ({})", |
| 82 | partition.getId(), partition.getVersion()); |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 83 | } else { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 84 | log.info("Failed to start server for partition {} ({})", |
| 85 | partition.getId(), partition.getVersion(), e); |
Madan Jampani | fc98177 | 2016-02-16 09:46:42 -0800 | [diff] [blame] | 86 | } |
| 87 | }).thenApply(v -> null); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 88 | } |
| 89 | |
| 90 | @Override |
| 91 | public CompletableFuture<Void> close() { |
Madan Jampani | 630e7ac | 2016-05-31 11:34:05 -0700 | [diff] [blame] | 92 | return server.shutdown(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 93 | } |
| 94 | |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 95 | /** |
| 96 | * Closes the server and exits the partition. |
| 97 | * @return future that is completed when the operation is complete |
| 98 | */ |
| 99 | public CompletableFuture<Void> closeAndExit() { |
Madan Jampani | 630e7ac | 2016-05-31 11:34:05 -0700 | [diff] [blame] | 100 | return server.leave(); |
Madan Jampani | 3354745 | 2016-02-29 16:45:04 -0800 | [diff] [blame] | 101 | } |
| 102 | |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 103 | /** |
| 104 | * Forks the existing partition into a new partition. |
| 105 | * |
| 106 | * @param version the version from which to fork the server |
| 107 | * @return future to be completed once the fork operation is complete |
| 108 | */ |
| 109 | public CompletableFuture<Void> fork(Version version) { |
| 110 | log.info("Forking server for partition {} ({}->{})", partition.getId(), version, partition.getVersion()); |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 111 | RaftServer.Builder builder = RaftServer.newBuilder(localMemberId) |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 112 | .withName(partition.getName(version)) |
| 113 | .withType(RaftMember.Type.PASSIVE) |
| 114 | .withProtocol(new RaftServerCommunicator( |
| 115 | partition.getName(version), |
| 116 | Serializer.using(StorageNamespaces.RAFT_PROTOCOL), |
| 117 | clusterCommunicator)) |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 118 | .withElectionTimeout(Duration.ofMillis(ELECTION_TIMEOUT_MILLIS)) |
| 119 | .withHeartbeatInterval(Duration.ofMillis(HEARTBEAT_INTERVAL_MILLIS)) |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 120 | .withStorage(RaftStorage.newBuilder() |
Jordan Halterman | 1920123 | 2017-09-12 17:20:26 -0700 | [diff] [blame] | 121 | .withStorageLevel(StorageLevel.MAPPED) |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 122 | .withSerializer(new AtomixSerializerAdapter(Serializer.using(StorageNamespaces.RAFT_STORAGE))) |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 123 | .withDirectory(partition.getDataFolder()) |
| 124 | .withMaxSegmentSize(MAX_SEGMENT_SIZE) |
| 125 | .build()); |
| 126 | StoragePartition.RAFT_SERVICES.forEach(builder::addService); |
| 127 | RaftServer server = builder.build(); |
| 128 | return server.join(partition.getMemberIds(version)) |
| 129 | .thenCompose(v -> server.shutdown()) |
| 130 | .thenCompose(v -> { |
| 131 | // Delete the cluster configuration file from the forked partition. |
| 132 | try { |
| 133 | Files.delete(new File(partition.getDataFolder(), "atomix.conf").toPath()); |
| 134 | } catch (IOException e) { |
| 135 | log.error("Failed to delete partition configuration: {}", e); |
| 136 | } |
| 137 | |
| 138 | // Build and bootstrap a new server. |
| 139 | this.server = buildServer(); |
| 140 | return this.server.bootstrap(); |
| 141 | }).whenComplete((r, e) -> { |
| 142 | if (e == null) { |
| 143 | log.info("Successfully forked server for partition {} ({}->{})", |
| 144 | partition.getId(), version, partition.getVersion()); |
| 145 | } else { |
| 146 | log.info("Failed to fork server for partition {} ({}->{})", |
| 147 | partition.getId(), version, partition.getVersion(), e); |
| 148 | } |
| 149 | }).thenApply(v -> null); |
| 150 | } |
| 151 | |
| 152 | private RaftServer buildServer() { |
| 153 | RaftServer.Builder builder = RaftServer.newBuilder(localMemberId) |
| 154 | .withName(partition.getName()) |
| 155 | .withProtocol(new RaftServerCommunicator( |
| 156 | partition.getName(), |
| 157 | Serializer.using(StorageNamespaces.RAFT_PROTOCOL), |
| 158 | clusterCommunicator)) |
| 159 | .withElectionTimeout(Duration.ofMillis(ELECTION_TIMEOUT_MILLIS)) |
| 160 | .withHeartbeatInterval(Duration.ofMillis(HEARTBEAT_INTERVAL_MILLIS)) |
| 161 | .withStorage(RaftStorage.newBuilder() |
| 162 | .withStorageLevel(StorageLevel.MAPPED) |
| 163 | .withSerializer(new AtomixSerializerAdapter(Serializer.using(StorageNamespaces.RAFT_STORAGE))) |
| 164 | .withDirectory(partition.getDataFolder()) |
Jordan Halterman | 035231e | 2017-07-18 08:39:07 -0700 | [diff] [blame] | 165 | .withMaxSegmentSize(MAX_SEGMENT_SIZE) |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 166 | .build()); |
| 167 | StoragePartition.RAFT_SERVICES.forEach(builder::addService); |
| 168 | return builder.build(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 169 | } |
| 170 | |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 171 | public CompletableFuture<Void> join(Collection<MemberId> otherMembers) { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 172 | log.info("Joining partition {} ({})", partition.getId(), partition.getVersion()); |
Madan Jampani | 630e7ac | 2016-05-31 11:34:05 -0700 | [diff] [blame] | 173 | server = buildServer(); |
| 174 | return server.join(otherMembers).whenComplete((r, e) -> { |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 175 | if (e == null) { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 176 | log.info("Successfully joined partition {} ({})", partition.getId(), partition.getVersion()); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 177 | } else { |
Jordan Halterman | 980a8c1 | 2017-09-22 18:01:19 -0700 | [diff] [blame^] | 178 | log.info("Failed to join partition {} ({})", partition.getId(), partition.getVersion(), e); |
Madan Jampani | f172d40 | 2016-03-04 00:56:38 -0800 | [diff] [blame] | 179 | } |
| 180 | }).thenApply(v -> null); |
| 181 | } |
| 182 | |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 183 | @Override |
| 184 | public boolean isOpen() { |
Madan Jampani | 65f24bb | 2016-03-15 15:16:18 -0700 | [diff] [blame] | 185 | return server.isRunning(); |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 186 | } |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 187 | |
| 188 | /** |
| 189 | * Returns the partition information. |
| 190 | * @return partition info |
| 191 | */ |
| 192 | public PartitionInfo info() { |
| 193 | return new StoragePartitionDetails(partition.getId(), |
Jordan Halterman | 2bf177c | 2017-06-29 01:49:08 -0700 | [diff] [blame] | 194 | server.cluster().getMembers(), |
| 195 | server.cluster().getMembers(), |
| 196 | server.cluster().getLeader(), |
| 197 | server.cluster().getTerm()).toPartitionInfo(); |
Madan Jampani | e14a09c | 2016-02-11 10:43:21 -0800 | [diff] [blame] | 198 | } |
Madan Jampani | 15b8ef5 | 2016-02-02 17:35:05 -0800 | [diff] [blame] | 199 | } |