| /* |
| * Copyright 2015-present Open Networking Foundation |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.onosproject.store.intent.impl; |
| |
| import com.google.common.collect.ImmutableList; |
| import org.apache.commons.lang.math.RandomUtils; |
| import org.apache.felix.scr.annotations.Activate; |
| import org.apache.felix.scr.annotations.Component; |
| import org.apache.felix.scr.annotations.Deactivate; |
| import org.apache.felix.scr.annotations.Modified; |
| import org.apache.felix.scr.annotations.Property; |
| import org.apache.felix.scr.annotations.Reference; |
| import org.apache.felix.scr.annotations.ReferenceCardinality; |
| import org.apache.felix.scr.annotations.Service; |
| import org.onlab.util.Backtrace; |
| import org.onlab.util.KryoNamespace; |
| import org.onosproject.cfg.ComponentConfigService; |
| import org.onosproject.cluster.ClusterService; |
| import org.onosproject.cluster.ControllerNode; |
| import org.onosproject.cluster.NodeId; |
| import org.onosproject.incubator.net.virtual.NetworkId; |
| import org.onosproject.incubator.net.virtual.VirtualNetworkIntent; |
| import org.onosproject.net.intent.Intent; |
| import org.onosproject.net.intent.IntentData; |
| import org.onosproject.net.intent.IntentEvent; |
| import org.onosproject.net.intent.IntentState; |
| import org.onosproject.net.intent.IntentStore; |
| import org.onosproject.net.intent.IntentStoreDelegate; |
| import org.onosproject.net.intent.Key; |
| import org.onosproject.net.intent.WorkPartitionService; |
| import org.onosproject.store.AbstractStore; |
| import org.onosproject.store.Timestamp; |
| import org.onosproject.store.serializers.KryoNamespaces; |
| import org.onosproject.store.service.EventuallyConsistentMap; |
| import org.onosproject.store.service.EventuallyConsistentMapBuilder; |
| import org.onosproject.store.service.EventuallyConsistentMapEvent; |
| import org.onosproject.store.service.EventuallyConsistentMapListener; |
| import org.onosproject.store.service.MultiValuedTimestamp; |
| import org.onosproject.store.service.StorageService; |
| import org.onosproject.store.service.WallClockTimestamp; |
| import org.osgi.service.component.ComponentContext; |
| import org.slf4j.Logger; |
| |
| import java.util.Collection; |
| import java.util.Dictionary; |
| import java.util.List; |
| import java.util.Objects; |
| import java.util.Optional; |
| import java.util.Properties; |
| import java.util.concurrent.atomic.AtomicLong; |
| import java.util.stream.Collectors; |
| |
| import static com.google.common.base.Preconditions.checkNotNull; |
| import static com.google.common.base.Strings.isNullOrEmpty; |
| import static org.onlab.util.Tools.get; |
| import static org.onosproject.net.intent.IntentState.PURGE_REQ; |
| import static org.slf4j.LoggerFactory.getLogger; |
| |
| /** |
| * Manages inventory of Intents in a distributed data store that uses optimistic |
| * replication and gossip based techniques. |
| */ |
| //FIXME we should listen for leadership changes. if the local instance has just |
| // ... become a leader, scan the pending map and process those |
| @Component(immediate = true) |
| @Service |
| public class GossipIntentStore |
| extends AbstractStore<IntentEvent, IntentStoreDelegate> |
| implements IntentStore { |
| |
| private final Logger log = getLogger(getClass()); |
| |
| private static final boolean PERSIST = false; |
| |
| // Map of intent key => current intent state |
| private EventuallyConsistentMap<Key, IntentData> currentMap; |
| |
| // Map of intent key => pending intent operation |
| private EventuallyConsistentMap<Key, IntentData> pendingMap; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected ComponentConfigService configService; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected ClusterService clusterService; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected StorageService storageService; |
| |
| @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) |
| protected WorkPartitionService partitionService; |
| |
| private final AtomicLong sequenceNumber = new AtomicLong(0); |
| |
| private EventuallyConsistentMapListener<Key, IntentData> |
| mapCurrentListener = new InternalCurrentListener(); |
| |
| private EventuallyConsistentMapListener<Key, IntentData> |
| mapPendingListener = new InternalPendingListener(); |
| |
| //Denotes the initial persistence value for this structure |
| private boolean initiallyPersistent = false; |
| |
| //TODO this is currently an experimental feature used for performance |
| // evalutaion, enabling persistence with persist the intents but they will |
| // not be reinstalled and network state will not be consistent with the |
| // intents on cluster restart |
| @Property(name = "persistenceEnabled", boolValue = PERSIST, |
| label = "EXPERIMENTAL: Enable intent persistence") |
| private boolean persistenceEnabled; |
| |
| |
| /** |
| * TimestampProvieder for currentMap. |
| * |
| * @param key Intent key |
| * @param data Intent data |
| * @return generated time stamp |
| */ |
| private Timestamp currentTimestampProvider(Key key, IntentData data) { |
| // vector timestamp consisting from 3 components |
| // (request timestamp, internal state, sequence #) |
| |
| // 2nd component required to avoid compilation result overwriting installation state |
| // note: above is likely to be a sign of design issue in transition to installation phase |
| // 3rd component required for generating new timestamp for removal.. |
| return new MultiValuedTimestamp<>( |
| Optional.ofNullable(data.version()).orElseGet(WallClockTimestamp::new), |
| new MultiValuedTimestamp<>(data.internalStateVersion(), |
| sequenceNumber.incrementAndGet())); |
| } |
| |
| @Activate |
| public void activate(ComponentContext context) { |
| configService.registerProperties(getClass()); |
| modified(context); |
| //TODO persistent intents must be reevaluated and the appropriate |
| //processing done here, current implementation is not functional |
| //and is for performance evaluation only |
| initiallyPersistent = persistenceEnabled; |
| KryoNamespace.Builder intentSerializer = KryoNamespace.newBuilder() |
| .register(KryoNamespaces.API) |
| .register(IntentData.class) |
| .register(VirtualNetworkIntent.class) |
| .register(NetworkId.class) |
| .register(MultiValuedTimestamp.class); |
| |
| EventuallyConsistentMapBuilder currentECMapBuilder = |
| storageService.<Key, IntentData>eventuallyConsistentMapBuilder() |
| .withName("intent-current") |
| .withSerializer(intentSerializer) |
| .withTimestampProvider(this::currentTimestampProvider) |
| .withPeerUpdateFunction((key, intentData) -> getPeerNodes(key, intentData)); |
| |
| EventuallyConsistentMapBuilder pendingECMapBuilder = |
| storageService.<Key, IntentData>eventuallyConsistentMapBuilder() |
| .withName("intent-pending") |
| .withSerializer(intentSerializer) |
| .withTimestampProvider((key, intentData) -> |
| /* |
| We always want to accept new values in the pending map, |
| so we should use a high performance logical clock. |
| */ |
| /* |
| TODO We use the wall clock for the time being, but |
| this could result in issues if there is clock skew |
| across instances. |
| */ |
| new MultiValuedTimestamp<>(new WallClockTimestamp(), System.nanoTime())) |
| .withPeerUpdateFunction((key, intentData) -> getPeerNodes(key, intentData)); |
| if (initiallyPersistent) { |
| currentECMapBuilder = currentECMapBuilder.withPersistence(); |
| pendingECMapBuilder = pendingECMapBuilder.withPersistence(); |
| } |
| currentMap = currentECMapBuilder.build(); |
| pendingMap = pendingECMapBuilder.build(); |
| |
| currentMap.addListener(mapCurrentListener); |
| pendingMap.addListener(mapPendingListener); |
| |
| log.info("Started"); |
| } |
| |
| @Deactivate |
| public void deactivate() { |
| if (initiallyPersistent && !persistenceEnabled) { |
| pendingMap.clear(); |
| currentMap.clear(); |
| log.debug("Persistent state has been purged"); |
| } |
| currentMap.removeListener(mapCurrentListener); |
| pendingMap.removeListener(mapPendingListener); |
| currentMap.destroy(); |
| pendingMap.destroy(); |
| |
| log.info("Stopped"); |
| } |
| |
| @Modified |
| public void modified(ComponentContext context) { |
| Dictionary<?, ?> properties = context != null ? context.getProperties() |
| : new Properties(); |
| try { |
| String s = get(properties, "persistenceEnabled"); |
| persistenceEnabled = isNullOrEmpty(s) ? PERSIST : |
| Boolean.parseBoolean(s.trim()); |
| } catch (Exception e) { |
| persistenceEnabled = initiallyPersistent; |
| log.error("Failed to retrieve the property value for persist," + |
| "defaulting to the initial setting of \"{}\"" + |
| "any persistent state will not be purged, if " + |
| "this occurred at startup changes made in this" + |
| "session will not be persisted to disk", |
| initiallyPersistent); |
| } |
| if (persistenceEnabled) { |
| //FIXME persistence is an experimental feature, warnings can be removed |
| //when the feature is completed |
| log.warn("Persistence is an experimental feature, it is not fully " + |
| "functional and is intended only for " + |
| "performance evaluation"); |
| } |
| if (!initiallyPersistent && !persistenceEnabled) { |
| log.info("Persistence is set to \"false\", this was the initial" + |
| " setting so no state will be purged or " + |
| "persisted"); |
| } else if (!initiallyPersistent && persistenceEnabled) { |
| log.info("Persistence is set to \"true\", entries will be begin " + |
| "to be persisted after restart"); |
| } else if (initiallyPersistent && !persistenceEnabled) { |
| log.info("Persistence is set to \"false\", all persistent state " + |
| "will be purged on next shutdown"); |
| } else { |
| log.info("Persistence is set to \"true\", entries from this and" + |
| " subsequent sessions will be persisted"); |
| } |
| |
| |
| } |
| |
| @Override |
| public long getIntentCount() { |
| return currentMap.size(); |
| } |
| |
| @Override |
| public Iterable<Intent> getIntents() { |
| return currentMap.values().stream() |
| .map(IntentData::intent) |
| .collect(Collectors.toList()); |
| } |
| |
| @Override |
| public Iterable<IntentData> getIntentData(boolean localOnly, long olderThan) { |
| if (localOnly || olderThan > 0) { |
| long now = System.currentTimeMillis(); |
| final WallClockTimestamp time = new WallClockTimestamp(now - olderThan); |
| return currentMap.values().stream() |
| .filter(data -> data.version().isOlderThan(time) && |
| (!localOnly || isMaster(data.key()))) |
| .collect(Collectors.toList()); |
| } |
| return currentMap.values(); |
| } |
| |
| @Override |
| public IntentState getIntentState(Key intentKey) { |
| IntentData data = currentMap.get(intentKey); |
| if (data != null) { |
| return data.state(); |
| } |
| return null; |
| } |
| |
| @Override |
| public List<Intent> getInstallableIntents(Key intentKey) { |
| IntentData data = currentMap.get(intentKey); |
| if (data != null) { |
| return data.installables(); |
| } |
| return ImmutableList.of(); |
| } |
| |
| @Override |
| public void write(IntentData newData) { |
| checkNotNull(newData); |
| |
| IntentData currentData = currentMap.get(newData.key()); |
| if (IntentData.isUpdateAcceptable(currentData, newData)) { |
| // Only the master is modifying the current state. Therefore assume |
| // this always succeeds |
| if (newData.state() == PURGE_REQ) { |
| if (currentData != null) { |
| if (log.isTraceEnabled()) { |
| log.trace("Purging {} in currentMap. {}@{}", |
| newData.key(), newData.state(), newData.version(), |
| new Backtrace()); |
| } |
| currentMap.remove(newData.key(), currentData); |
| } else { |
| log.info("Gratuitous purge request for intent: {}", newData.key()); |
| } |
| } else { |
| if (log.isTraceEnabled()) { |
| log.trace("Putting {} in currentMap. {}@{}", |
| newData.key(), newData.state(), newData.version(), |
| new Backtrace()); |
| } |
| currentMap.put(newData.key(), IntentData.copy(newData)); |
| } |
| } else { |
| log.debug("Update for {} not acceptable from:\n{}\nto:\n{}", |
| newData.key(), currentData, newData); |
| } |
| // Remove the intent data from the pending map if the newData is more |
| // recent or equal to the existing entry. No matter if it is an acceptable |
| // update or not |
| Key key = newData.key(); |
| IntentData existingValue = pendingMap.get(key); |
| |
| if (existingValue == null) { |
| return; |
| } |
| |
| if (!existingValue.version().isNewerThan(newData.version())) { |
| pendingMap.remove(key, existingValue); |
| } else { |
| log.debug("{} in pending map was newer, leaving it there", key); |
| } |
| } |
| |
| private Collection<NodeId> getPeerNodes(Key key, IntentData data) { |
| NodeId master = partitionService.getLeader(key, Key::hash); |
| NodeId origin = (data != null) ? data.origin() : null; |
| if (data != null && (master == null || origin == null)) { |
| log.debug("Intent {} missing master and/or origin; master = {}, origin = {}", |
| key, master, origin); |
| } |
| |
| NodeId me = clusterService.getLocalNode().id(); |
| boolean isMaster = Objects.equals(master, me); |
| boolean isOrigin = Objects.equals(origin, me); |
| if (isMaster && isOrigin) { |
| return getRandomNode(); |
| } else if (isMaster) { |
| return origin != null ? ImmutableList.of(origin) : getRandomNode(); |
| } else if (isOrigin) { |
| return master != null ? ImmutableList.of(master) : getRandomNode(); |
| } else { |
| log.warn("No master or origin for intent {}", key); |
| return master != null ? ImmutableList.of(master) : getRandomNode(); |
| } |
| } |
| |
| private List<NodeId> getRandomNode() { |
| NodeId me = clusterService.getLocalNode().id(); |
| List<NodeId> nodes = clusterService.getNodes().stream() |
| .map(ControllerNode::id) |
| .filter(node -> !Objects.equals(node, me)) |
| .collect(Collectors.toList()); |
| if (nodes.isEmpty()) { |
| return ImmutableList.of(); |
| } |
| return ImmutableList.of(nodes.get(RandomUtils.nextInt(nodes.size()))); |
| } |
| |
| @Override |
| public void batchWrite(Iterable<IntentData> updates) { |
| updates.forEach(this::write); |
| } |
| |
| @Override |
| public Intent getIntent(Key key) { |
| IntentData data = currentMap.get(key); |
| if (data != null) { |
| return data.intent(); |
| } |
| return null; |
| } |
| |
| @Override |
| public IntentData getIntentData(Key key) { |
| IntentData current = currentMap.get(key); |
| if (current == null) { |
| return null; |
| } |
| return IntentData.copy(current); |
| } |
| |
| @Override |
| public void addPending(IntentData data) { |
| checkNotNull(data); |
| if (data.version() == null) { |
| // Copy IntentData including request state in this way we can |
| // avoid the creation of Intents with state == request, which can |
| // be problematic if the Intent state is different from *REQ |
| // {INSTALL_, WITHDRAW_ and PURGE_}. |
| pendingMap.put(data.key(), IntentData.assign(data, |
| new WallClockTimestamp(), |
| clusterService.getLocalNode().id())); |
| } else { |
| pendingMap.compute(data.key(), (key, existingValue) -> { |
| if (existingValue == null || existingValue.version().isOlderThan(data.version())) { |
| return IntentData.assign(data, data.version(), clusterService.getLocalNode().id()); |
| } else { |
| return existingValue; |
| } |
| }); |
| } |
| |
| } |
| |
| @Override |
| public boolean isMaster(Key intentKey) { |
| return partitionService.isMine(intentKey, Key::hash); |
| } |
| |
| @Override |
| public Iterable<Intent> getPending() { |
| return pendingMap.values().stream() |
| .map(IntentData::intent) |
| .collect(Collectors.toList()); |
| } |
| |
| @Override |
| public Iterable<IntentData> getPendingData() { |
| return pendingMap.values(); |
| } |
| |
| @Override |
| public IntentData getPendingData(Key intentKey) { |
| return pendingMap.get(intentKey); |
| } |
| |
| @Override |
| public Iterable<IntentData> getPendingData(boolean localOnly, long olderThan) { |
| long now = System.currentTimeMillis(); |
| final WallClockTimestamp time = new WallClockTimestamp(now - olderThan); |
| return pendingMap.values().stream() |
| .filter(data -> data.version().isOlderThan(time) && |
| (!localOnly || isMaster(data.key()))) |
| .collect(Collectors.toList()); |
| } |
| |
| private final class InternalCurrentListener implements |
| EventuallyConsistentMapListener<Key, IntentData> { |
| @Override |
| public void event(EventuallyConsistentMapEvent<Key, IntentData> event) { |
| IntentData intentData = event.value(); |
| if (event.type() == EventuallyConsistentMapEvent.Type.PUT) { |
| // The current intents map has been updated. If we are master for |
| // this intent's partition, notify the Manager that it should |
| // emit notifications about updated tracked resources. |
| if (delegate != null && isMaster(event.value().intent().key())) { |
| delegate.onUpdate(IntentData.copy(intentData)); // copy for safety, likely unnecessary |
| } |
| IntentEvent.getEvent(intentData).ifPresent(e -> notifyDelegate(e)); |
| } |
| } |
| } |
| |
| private final class InternalPendingListener implements |
| EventuallyConsistentMapListener<Key, IntentData> { |
| @Override |
| public void event( |
| EventuallyConsistentMapEvent<Key, IntentData> event) { |
| if (event.type() == EventuallyConsistentMapEvent.Type.PUT) { |
| // The pending intents map has been updated. If we are master for |
| // this intent's partition, notify the Manager that it should do |
| // some work. |
| if (isMaster(event.value().intent().key())) { |
| if (delegate != null) { |
| delegate.process(IntentData.copy(event.value())); |
| } |
| } |
| |
| IntentEvent.getEvent(event.value()).ifPresent(e -> notifyDelegate(e)); |
| } |
| } |
| } |
| |
| } |
| |