Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2014 Open Networking Laboratory |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package org.onlab.onos.store.service.impl; |
| 18 | |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 19 | import static org.onlab.util.Tools.namedThreads; |
| 20 | |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 21 | import java.io.IOException; |
| 22 | import java.util.HashMap; |
| 23 | import java.util.Map; |
| 24 | import java.util.Objects; |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 25 | import java.util.concurrent.ExecutorService; |
| 26 | import java.util.concurrent.Executors; |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 27 | import java.util.concurrent.TimeUnit; |
| 28 | import java.util.concurrent.atomic.AtomicBoolean; |
| 29 | |
| 30 | import net.jodah.expiringmap.ExpiringMap; |
| 31 | import net.jodah.expiringmap.ExpiringMap.ExpirationListener; |
| 32 | import net.jodah.expiringmap.ExpiringMap.ExpirationPolicy; |
| 33 | import net.kuujo.copycat.cluster.Member; |
| 34 | import net.kuujo.copycat.event.EventHandler; |
| 35 | import net.kuujo.copycat.event.LeaderElectEvent; |
| 36 | |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 37 | import org.onlab.onos.cluster.ControllerNode; |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 38 | import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService; |
| 39 | import org.onlab.onos.store.cluster.messaging.ClusterMessage; |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 40 | import org.onlab.onos.store.service.DatabaseService; |
| 41 | import org.onlab.onos.store.service.VersionedValue; |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 42 | import org.onlab.onos.store.service.impl.DatabaseStateMachine.State; |
| 43 | import org.onlab.onos.store.service.impl.DatabaseStateMachine.TableMetadata; |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 44 | import org.slf4j.Logger; |
| 45 | import org.slf4j.LoggerFactory; |
| 46 | |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 47 | import com.google.common.base.MoreObjects; |
| 48 | |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 49 | /** |
| 50 | * Plugs into the database update stream and track the TTL of entries added to |
| 51 | * the database. For tables with pre-configured finite TTL, this class has |
| 52 | * mechanisms for expiring (deleting) old, expired entries from the database. |
| 53 | */ |
| 54 | public class DatabaseEntryExpirationTracker implements |
| 55 | DatabaseUpdateEventListener, EventHandler<LeaderElectEvent> { |
| 56 | |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 57 | private static final ExecutorService THREAD_POOL = |
| 58 | Executors.newCachedThreadPool(namedThreads("database-stale-entry-expirer-%d")); |
| 59 | |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 60 | private final Logger log = LoggerFactory.getLogger(getClass()); |
| 61 | |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 62 | private final DatabaseService databaseService; |
| 63 | private final ClusterCommunicationService clusterCommunicator; |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 64 | |
| 65 | private final Member localMember; |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 66 | private final ControllerNode localNode; |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 67 | private final AtomicBoolean isLocalMemberLeader = new AtomicBoolean(false); |
| 68 | |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 69 | private final Map<String, Map<DatabaseRow, Long>> tableEntryExpirationMap = new HashMap<>(); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 70 | |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 71 | private final ExpirationListener<DatabaseRow, Long> expirationObserver = new ExpirationObserver(); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 72 | |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 73 | DatabaseEntryExpirationTracker( |
| 74 | Member localMember, |
| 75 | ControllerNode localNode, |
| 76 | ClusterCommunicationService clusterCommunicator, |
| 77 | DatabaseService databaseService) { |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 78 | this.localMember = localMember; |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 79 | this.localNode = localNode; |
| 80 | this.clusterCommunicator = clusterCommunicator; |
| 81 | this.databaseService = databaseService; |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 82 | } |
| 83 | |
| 84 | @Override |
| 85 | public void tableModified(TableModificationEvent event) { |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 86 | log.debug("{}: Received {}", localNode.id(), event); |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 87 | |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 88 | if (!tableEntryExpirationMap.containsKey(event.tableName())) { |
| 89 | return; |
| 90 | } |
| 91 | |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 92 | Map<DatabaseRow, Long> map = tableEntryExpirationMap.get(event.tableName()); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 93 | DatabaseRow row = new DatabaseRow(event.tableName(), event.key()); |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 94 | Long eventVersion = event.value().version(); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 95 | |
| 96 | switch (event.type()) { |
| 97 | case ROW_DELETED: |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 98 | map.remove(row, eventVersion); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 99 | if (isLocalMemberLeader.get()) { |
| 100 | try { |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 101 | log.debug("Broadcasting {} to the entire cluster", event); |
| 102 | clusterCommunicator.broadcastIncludeSelf(new ClusterMessage( |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 103 | localNode.id(), DatabaseStateMachine.DATABASE_UPDATE_EVENTS, |
Yuta HIGUCHI | 91768e3 | 2014-11-22 05:06:35 -0800 | [diff] [blame] | 104 | ClusterMessagingProtocol.DB_SERIALIZER.encode(event))); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 105 | } catch (IOException e) { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 106 | log.error("Failed to broadcast a database row deleted event.", e); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 107 | } |
| 108 | } |
| 109 | break; |
| 110 | case ROW_ADDED: |
| 111 | case ROW_UPDATED: |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 112 | // To account for potential reordering of notifications, |
| 113 | // check to make sure we are replacing an old version with a new version |
| 114 | Long currentVersion = map.get(row); |
| 115 | if (currentVersion == null || currentVersion < eventVersion) { |
| 116 | map.put(row, eventVersion); |
| 117 | } |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 118 | break; |
| 119 | default: |
| 120 | break; |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | @Override |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 125 | public void tableCreated(TableMetadata metadata) { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 126 | log.debug("Received a table created event {}", metadata); |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 127 | if (metadata.expireOldEntries()) { |
| 128 | tableEntryExpirationMap.put(metadata.tableName(), ExpiringMap.builder() |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 129 | .expiration(metadata.ttlMillis(), TimeUnit.MILLISECONDS) |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 130 | .expirationListener(expirationObserver) |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 131 | .expirationPolicy(ExpirationPolicy.CREATED).build()); |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | @Override |
| 136 | public void tableDeleted(String tableName) { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 137 | log.debug("Received a table deleted event for table ({})", tableName); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 138 | tableEntryExpirationMap.remove(tableName); |
| 139 | } |
| 140 | |
| 141 | private class ExpirationObserver implements |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 142 | ExpirationListener<DatabaseRow, Long> { |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 143 | @Override |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 144 | public void expired(DatabaseRow row, Long version) { |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 145 | THREAD_POOL.submit(new ExpirationTask(row, version)); |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | private class ExpirationTask implements Runnable { |
| 150 | |
| 151 | private final DatabaseRow row; |
| 152 | private final Long version; |
| 153 | |
| 154 | public ExpirationTask(DatabaseRow row, Long version) { |
| 155 | this.row = row; |
| 156 | this.version = version; |
| 157 | } |
| 158 | |
| 159 | @Override |
| 160 | public void run() { |
| 161 | log.debug("Received an expiration event for {}, version: {}", row, version); |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 162 | Map<DatabaseRow, Long> map = tableEntryExpirationMap.get(row.tableName); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 163 | try { |
| 164 | if (isLocalMemberLeader.get()) { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 165 | if (!databaseService.removeIfVersionMatches(row.tableName, |
| 166 | row.key, version)) { |
| 167 | log.info("Entry in database was updated right before its expiration."); |
| 168 | } else { |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 169 | log.debug("Successfully expired old entry with key ({}) from table ({})", |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 170 | row.key, row.tableName); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 171 | } |
| 172 | } else { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 173 | // Only the current leader will expire keys from database. |
| 174 | // Everyone else function as standby just in case they need to take over |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 175 | if (map != null) { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 176 | map.putIfAbsent(row, version); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 177 | } |
| 178 | } |
| 179 | |
| 180 | } catch (Exception e) { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 181 | log.warn("Failed to delete entry from the database after ttl " |
| 182 | + "expiration. Operation will be retried.", e); |
| 183 | map.putIfAbsent(row, version); |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 184 | } |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | @Override |
| 189 | public void handle(LeaderElectEvent event) { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 190 | isLocalMemberLeader.set(localMember.equals(event.leader())); |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 191 | if (isLocalMemberLeader.get()) { |
| 192 | log.info("{} is now the leader of Raft cluster", localNode.id()); |
| 193 | } |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 194 | } |
| 195 | |
| 196 | /** |
| 197 | * Wrapper class for a database row identifier. |
| 198 | */ |
| 199 | private class DatabaseRow { |
| 200 | |
| 201 | String tableName; |
| 202 | String key; |
| 203 | |
| 204 | public DatabaseRow(String tableName, String key) { |
| 205 | this.tableName = tableName; |
| 206 | this.key = key; |
| 207 | } |
| 208 | |
| 209 | @Override |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 210 | public String toString() { |
| 211 | return MoreObjects.toStringHelper(getClass()) |
| 212 | .add("tableName", tableName) |
| 213 | .add("key", key) |
| 214 | .toString(); |
| 215 | } |
| 216 | |
| 217 | @Override |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 218 | public boolean equals(Object obj) { |
| 219 | if (this == obj) { |
| 220 | return true; |
| 221 | } |
| 222 | if (!(obj instanceof DatabaseRow)) { |
| 223 | return false; |
| 224 | } |
| 225 | DatabaseRow that = (DatabaseRow) obj; |
| 226 | |
| 227 | return Objects.equals(this.tableName, that.tableName) |
| 228 | && Objects.equals(this.key, that.key); |
| 229 | } |
| 230 | |
| 231 | @Override |
| 232 | public int hashCode() { |
| 233 | return Objects.hash(tableName, key); |
| 234 | } |
| 235 | } |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 236 | |
| 237 | @Override |
| 238 | public void snapshotInstalled(State state) { |
| 239 | if (!tableEntryExpirationMap.isEmpty()) { |
| 240 | return; |
| 241 | } |
Madan Jampani | a88d1f5 | 2014-11-14 16:45:24 -0800 | [diff] [blame] | 242 | log.debug("Received a snapshot installed notification"); |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 243 | for (String tableName : state.getTableNames()) { |
| 244 | |
| 245 | TableMetadata metadata = state.getTableMetadata(tableName); |
| 246 | if (!metadata.expireOldEntries()) { |
| 247 | continue; |
| 248 | } |
| 249 | |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 250 | Map<DatabaseRow, Long> tableExpirationMap = ExpiringMap.builder() |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 251 | .expiration(metadata.ttlMillis(), TimeUnit.MILLISECONDS) |
| 252 | .expirationListener(expirationObserver) |
| 253 | .expirationPolicy(ExpirationPolicy.CREATED).build(); |
| 254 | for (Map.Entry<String, VersionedValue> entry : state.getTable(tableName).entrySet()) { |
Madan Jampani | f5d263b | 2014-11-13 10:04:40 -0800 | [diff] [blame] | 255 | tableExpirationMap.put(new DatabaseRow(tableName, entry.getKey()), entry.getValue().version()); |
Madan Jampani | def2c65 | 2014-11-12 13:50:10 -0800 | [diff] [blame] | 256 | } |
| 257 | |
| 258 | tableEntryExpirationMap.put(tableName, tableExpirationMap); |
| 259 | } |
| 260 | } |
Madan Jampani | 9b37d57 | 2014-11-12 11:53:24 -0800 | [diff] [blame] | 261 | } |