blob: e7c27bccf7680a35b7a72f36ea0dca8001f9fd84 [file] [log] [blame]
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -07001/*
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -08002 * Copyright 2015-2016 Open Networking Laboratory
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -07003 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
Sho SHIMIZUe18cb122016-02-22 21:04:56 -080016package org.onosproject.store.resource.impl;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070017
18import com.google.common.annotations.Beta;
Sho SHIMIZUe7db6142015-11-04 11:24:22 -080019import com.google.common.collect.ImmutableList;
Sho SHIMIZU83258ae2016-01-29 17:39:07 -080020import com.google.common.collect.ImmutableSet;
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -080021import com.google.common.collect.Maps;
Sho SHIMIZU67c90102016-02-23 10:49:58 -080022import com.google.common.collect.Sets;
Madan Jampani3780d4b2016-04-04 18:18:24 -070023
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070024import org.apache.felix.scr.annotations.Activate;
25import org.apache.felix.scr.annotations.Component;
26import org.apache.felix.scr.annotations.Reference;
27import org.apache.felix.scr.annotations.ReferenceCardinality;
28import org.apache.felix.scr.annotations.Service;
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -080029import org.onlab.util.GuavaCollectors;
Thomas Vachuska762a2d82016-01-04 10:25:20 -080030import org.onlab.util.Tools;
Sho SHIMIZUe18cb122016-02-22 21:04:56 -080031import org.onosproject.net.resource.ContinuousResource;
32import org.onosproject.net.resource.ContinuousResourceId;
33import org.onosproject.net.resource.DiscreteResource;
34import org.onosproject.net.resource.DiscreteResourceId;
35import org.onosproject.net.resource.ResourceAllocation;
36import org.onosproject.net.resource.ResourceConsumer;
37import org.onosproject.net.resource.ResourceEvent;
38import org.onosproject.net.resource.ResourceId;
39import org.onosproject.net.resource.Resource;
40import org.onosproject.net.resource.ResourceStore;
41import org.onosproject.net.resource.ResourceStoreDelegate;
42import org.onosproject.net.resource.Resources;
Sho SHIMIZUfa62b472015-11-02 17:35:46 -080043import org.onosproject.store.AbstractStore;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070044import org.onosproject.store.serializers.KryoNamespaces;
Madan Jampani3780d4b2016-04-04 18:18:24 -070045import org.onosproject.store.service.CommitStatus;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070046import org.onosproject.store.service.ConsistentMap;
Thomas Vachuska762a2d82016-01-04 10:25:20 -080047import org.onosproject.store.service.ConsistentMapException;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070048import org.onosproject.store.service.Serializer;
49import org.onosproject.store.service.StorageService;
50import org.onosproject.store.service.TransactionContext;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070051import org.onosproject.store.service.TransactionalMap;
52import org.onosproject.store.service.Versioned;
53import org.slf4j.Logger;
54import org.slf4j.LoggerFactory;
55
Sho SHIMIZUba41fc12015-08-12 15:43:22 -070056import java.util.Arrays;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070057import java.util.Collection;
Sho SHIMIZU69420fe2016-02-09 15:01:07 -080058import java.util.LinkedHashMap;
Sho SHIMIZUba41fc12015-08-12 15:43:22 -070059import java.util.LinkedHashSet;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070060import java.util.List;
61import java.util.Map;
62import java.util.Optional;
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -080063import java.util.Set;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070064import java.util.stream.Collectors;
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -080065import java.util.stream.Stream;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070066
67import static com.google.common.base.Preconditions.checkArgument;
68import static com.google.common.base.Preconditions.checkNotNull;
Sho SHIMIZUe18cb122016-02-22 21:04:56 -080069import static org.onosproject.net.resource.ResourceEvent.Type.*;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070070
71/**
72 * Implementation of ResourceStore using TransactionalMap.
73 */
Sho SHIMIZU9a2b8292015-10-28 13:00:16 -070074@Component(immediate = true)
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070075@Service
76@Beta
Sho SHIMIZUfa62b472015-11-02 17:35:46 -080077public class ConsistentResourceStore extends AbstractStore<ResourceEvent, ResourceStoreDelegate>
78 implements ResourceStore {
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070079 private static final Logger log = LoggerFactory.getLogger(ConsistentResourceStore.class);
80
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -080081 private static final String DISCRETE_CONSUMER_MAP = "onos-discrete-consumers";
82 private static final String CONTINUOUS_CONSUMER_MAP = "onos-continuous-consumers";
Sho SHIMIZUba41fc12015-08-12 15:43:22 -070083 private static final String CHILD_MAP = "onos-resource-children";
84 private static final Serializer SERIALIZER = Serializer.using(
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -080085 Arrays.asList(KryoNamespaces.BASIC, KryoNamespaces.API),
86 ContinuousResourceAllocation.class);
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070087
Thomas Vachuska762a2d82016-01-04 10:25:20 -080088 // TODO: We should provide centralized values for this
89 private static final int MAX_RETRIES = 5;
90 private static final int RETRY_DELAY = 1_000; // millis
91
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070092 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
93 protected StorageService service;
94
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -080095 private ConsistentMap<DiscreteResourceId, ResourceConsumer> discreteConsumers;
96 private ConsistentMap<ContinuousResourceId, ContinuousResourceAllocation> continuousConsumers;
97 private ConsistentMap<DiscreteResourceId, Set<Resource>> childMap;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -070098
99 @Activate
100 public void activate() {
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800101 discreteConsumers = service.<DiscreteResourceId, ResourceConsumer>consistentMapBuilder()
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800102 .withName(DISCRETE_CONSUMER_MAP)
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700103 .withSerializer(SERIALIZER)
104 .build();
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800105 continuousConsumers = service.<ContinuousResourceId, ContinuousResourceAllocation>consistentMapBuilder()
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800106 .withName(CONTINUOUS_CONSUMER_MAP)
107 .withSerializer(SERIALIZER)
108 .build();
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800109 childMap = service.<DiscreteResourceId, Set<Resource>>consistentMapBuilder()
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700110 .withName(CHILD_MAP)
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700111 .withSerializer(SERIALIZER)
112 .build();
Sho SHIMIZUe7db6142015-11-04 11:24:22 -0800113
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800114 Tools.retryable(() -> childMap.put(Resource.ROOT.id(), new LinkedHashSet<>()),
Thomas Vachuska762a2d82016-01-04 10:25:20 -0800115 ConsistentMapException.class, MAX_RETRIES, RETRY_DELAY);
Madan Jampanic7f49f92015-12-10 11:35:06 -0800116 log.info("Started");
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700117 }
118
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800119 // Computational complexity: O(1) if the resource is discrete type.
120 // O(n) if the resource is continuous type where n is the number of the existing allocations for the resource
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700121 @Override
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800122 public List<ResourceAllocation> getResourceAllocations(ResourceId id) {
123 checkNotNull(id);
124 checkArgument(id instanceof DiscreteResourceId || id instanceof ContinuousResourceId);
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700125
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800126 if (id instanceof DiscreteResourceId) {
127 return getResourceAllocations((DiscreteResourceId) id);
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800128 } else {
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800129 return getResourceAllocations((ContinuousResourceId) id);
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800130 }
131 }
132
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800133 // computational complexity: O(1)
134 private List<ResourceAllocation> getResourceAllocations(DiscreteResourceId resource) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800135 Versioned<ResourceConsumer> consumer = discreteConsumers.get(resource);
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700136 if (consumer == null) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800137 return ImmutableList.of();
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700138 }
139
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800140 return ImmutableList.of(new ResourceAllocation(Resources.discrete(resource).resource(), consumer.value()));
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800141 }
142
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800143 // computational complexity: O(n) where n is the number of the existing allocations for the resource
144 private List<ResourceAllocation> getResourceAllocations(ContinuousResourceId resource) {
145 Versioned<ContinuousResourceAllocation> allocations = continuousConsumers.get(resource);
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800146 if (allocations == null) {
147 return ImmutableList.of();
148 }
149
150 return allocations.value().allocations().stream()
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800151 .filter(x -> x.resource().id().equals(resource))
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800152 .collect(GuavaCollectors.toImmutableList());
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700153 }
154
155 @Override
Jonathan Hart56151262016-02-11 09:48:50 -0800156 public boolean register(List<Resource> resources) {
Sho SHIMIZU83e17a02015-08-20 14:07:05 -0700157 checkNotNull(resources);
HIGUCHI Yuta6f828c32016-01-20 18:11:05 -0800158 if (log.isTraceEnabled()) {
159 resources.forEach(r -> log.trace("registering {}", r));
160 }
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700161
162 TransactionContext tx = service.transactionContextBuilder().build();
163 tx.begin();
164
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800165 TransactionalMap<DiscreteResourceId, Set<Resource>> childTxMap =
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800166 tx.getTransactionalMap(CHILD_MAP, SERIALIZER);
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700167
Sho SHIMIZU69420fe2016-02-09 15:01:07 -0800168 // the order is preserved by LinkedHashMap
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800169 Map<DiscreteResource, List<Resource>> resourceMap = resources.stream()
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800170 .filter(x -> x.parent().isPresent())
Sho SHIMIZU69420fe2016-02-09 15:01:07 -0800171 .collect(Collectors.groupingBy(x -> x.parent().get(), LinkedHashMap::new, Collectors.toList()));
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700172
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800173 for (Map.Entry<DiscreteResource, List<Resource>> entry: resourceMap.entrySet()) {
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800174 if (!lookup(childTxMap, entry.getKey().id()).isPresent()) {
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800175 return abortTransaction(tx);
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700176 }
177
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800178 if (!appendValues(childTxMap, entry.getKey().id(), entry.getValue())) {
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800179 return abortTransaction(tx);
180 }
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700181 }
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800182
Madan Jampani3780d4b2016-04-04 18:18:24 -0700183 return tx.commit().whenComplete((status, error) -> {
184 if (status == CommitStatus.SUCCESS) {
185 log.trace("Transaction commit succeeded on registration: resources={}", resources);
186 List<ResourceEvent> events = resources.stream()
187 .filter(x -> x.parent().isPresent())
188 .map(x -> new ResourceEvent(RESOURCE_ADDED, x))
189 .collect(Collectors.toList());
190 notifyDelegate(events);
191 } else {
192 log.warn("Transaction commit failed on registration", error);
193 }
194 }).join() == CommitStatus.SUCCESS;
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700195 }
196
197 @Override
Jonathan Hart56151262016-02-11 09:48:50 -0800198 public boolean unregister(List<ResourceId> ids) {
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800199 checkNotNull(ids);
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700200
201 TransactionContext tx = service.transactionContextBuilder().build();
202 tx.begin();
203
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800204 TransactionalMap<DiscreteResourceId, Set<Resource>> childTxMap =
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800205 tx.getTransactionalMap(CHILD_MAP, SERIALIZER);
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800206 TransactionalMap<DiscreteResourceId, ResourceConsumer> discreteConsumerTxMap =
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800207 tx.getTransactionalMap(DISCRETE_CONSUMER_MAP, SERIALIZER);
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800208 TransactionalMap<ContinuousResourceId, ContinuousResourceAllocation> continuousConsumerTxMap =
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800209 tx.getTransactionalMap(CONTINUOUS_CONSUMER_MAP, SERIALIZER);
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700210
Sho SHIMIZU7d54d9c2016-02-17 13:58:46 -0800211 // Look up resources by resource IDs
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800212 List<Resource> resources = ids.stream()
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800213 .filter(x -> x.parent().isPresent())
Sho SHIMIZU7d54d9c2016-02-17 13:58:46 -0800214 .map(x -> {
215 // avoid access to consistent map in the case of discrete resource
216 if (x instanceof DiscreteResourceId) {
217 return Optional.of(Resources.discrete((DiscreteResourceId) x).resource());
218 } else {
219 return lookup(childTxMap, x);
220 }
221 })
HIGUCHI Yuta315179a2016-02-18 14:01:22 -0800222 .filter(Optional::isPresent)
223 .map(Optional::get)
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800224 .collect(Collectors.toList());
Sho SHIMIZU69420fe2016-02-09 15:01:07 -0800225 // the order is preserved by LinkedHashMap
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800226 Map<DiscreteResourceId, List<Resource>> resourceMap = resources.stream()
Sho SHIMIZU69420fe2016-02-09 15:01:07 -0800227 .collect(Collectors.groupingBy(x -> x.parent().get().id(), LinkedHashMap::new, Collectors.toList()));
Sho SHIMIZU83e17a02015-08-20 14:07:05 -0700228
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800229 // even if one of the resources is allocated to a consumer,
230 // all unregistrations are regarded as failure
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800231 for (Map.Entry<DiscreteResourceId, List<Resource>> entry: resourceMap.entrySet()) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800232 boolean allocated = entry.getValue().stream().anyMatch(x -> {
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800233 if (x instanceof DiscreteResource) {
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800234 return discreteConsumerTxMap.get(((DiscreteResource) x).id()) != null;
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800235 } else if (x instanceof ContinuousResource) {
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800236 ContinuousResourceAllocation allocations =
237 continuousConsumerTxMap.get(((ContinuousResource) x).id());
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800238 return allocations != null && !allocations.allocations().isEmpty();
239 } else {
240 return false;
241 }
242 });
243 if (allocated) {
HIGUCHI Yuta5b6dfba2016-01-27 14:43:41 -0800244 log.warn("Failed to unregister {}: allocation exists", entry.getKey());
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800245 return abortTransaction(tx);
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700246 }
247
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800248 if (!removeValues(childTxMap, entry.getKey(), entry.getValue())) {
HIGUCHI Yuta6acdfd02016-02-18 10:39:43 -0800249 log.warn("Failed to unregister {}: Failed to remove {} values.",
250 entry.getKey(), entry.getValue().size());
251 log.debug("Failed to unregister {}: Failed to remove values: {}",
HIGUCHI Yuta5b6dfba2016-01-27 14:43:41 -0800252 entry.getKey(), entry.getValue());
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800253 return abortTransaction(tx);
254 }
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700255 }
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800256
Madan Jampani3780d4b2016-04-04 18:18:24 -0700257 return tx.commit().whenComplete((status, error) -> {
258 if (status == CommitStatus.SUCCESS) {
259 List<ResourceEvent> events = resources.stream()
260 .filter(x -> x.parent().isPresent())
261 .map(x -> new ResourceEvent(RESOURCE_REMOVED, x))
262 .collect(Collectors.toList());
263 notifyDelegate(events);
264 } else {
265 log.warn("Failed to unregister {}: Commit failed.", ids, error);
266 }
267 }).join() == CommitStatus.SUCCESS;
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700268 }
269
270 @Override
Jonathan Hart56151262016-02-11 09:48:50 -0800271 public boolean allocate(List<Resource> resources, ResourceConsumer consumer) {
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700272 checkNotNull(resources);
273 checkNotNull(consumer);
274
275 TransactionContext tx = service.transactionContextBuilder().build();
276 tx.begin();
277
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800278 TransactionalMap<DiscreteResourceId, Set<Resource>> childTxMap =
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800279 tx.getTransactionalMap(CHILD_MAP, SERIALIZER);
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800280 TransactionalMap<DiscreteResourceId, ResourceConsumer> discreteConsumerTxMap =
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800281 tx.getTransactionalMap(DISCRETE_CONSUMER_MAP, SERIALIZER);
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800282 TransactionalMap<ContinuousResourceId, ContinuousResourceAllocation> continuousConsumerTxMap =
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800283 tx.getTransactionalMap(CONTINUOUS_CONSUMER_MAP, SERIALIZER);
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700284
Sho SHIMIZU8fa670a2016-01-14 11:17:18 -0800285 for (Resource resource: resources) {
Sho SHIMIZU171a9382016-02-15 13:56:34 -0800286 // if the resource is not registered, then abort
287 Optional<Resource> lookedUp = lookup(childTxMap, resource.id());
288 if (!lookedUp.isPresent()) {
289 return abortTransaction(tx);
290 }
Sho SHIMIZUd29847f2015-08-13 09:10:59 -0700291
Sho SHIMIZU171a9382016-02-15 13:56:34 -0800292 if (resource instanceof DiscreteResource) {
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800293 ResourceConsumer oldValue = discreteConsumerTxMap.put(((DiscreteResource) resource).id(), consumer);
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800294 if (oldValue != null) {
295 return abortTransaction(tx);
296 }
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800297 } else if (resource instanceof ContinuousResource) {
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800298 // Down cast: this must be safe as ContinuousResource is associated with ContinuousResourceId
299 ContinuousResource continuous = (ContinuousResource) lookedUp.get();
300 ContinuousResourceAllocation allocations = continuousConsumerTxMap.get(continuous.id());
301 if (!hasEnoughResource(continuous, (ContinuousResource) resource, allocations)) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800302 return abortTransaction(tx);
303 }
304
305 boolean success = appendValue(continuousConsumerTxMap,
Naoki Shiota3bac1c32016-03-31 17:46:45 -0700306 continuous, new ResourceAllocation(resource, consumer));
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800307 if (!success) {
308 return abortTransaction(tx);
309 }
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800310 }
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700311 }
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800312
Madan Jampani3780d4b2016-04-04 18:18:24 -0700313 return tx.commit().join() == CommitStatus.SUCCESS;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700314 }
315
316 @Override
Sho SHIMIZUfc64ffe2016-02-10 20:11:09 -0800317 public boolean release(List<ResourceAllocation> allocations) {
318 checkNotNull(allocations);
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700319
320 TransactionContext tx = service.transactionContextBuilder().build();
321 tx.begin();
322
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800323 TransactionalMap<DiscreteResourceId, ResourceConsumer> discreteConsumerTxMap =
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800324 tx.getTransactionalMap(DISCRETE_CONSUMER_MAP, SERIALIZER);
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800325 TransactionalMap<ContinuousResourceId, ContinuousResourceAllocation> continuousConsumerTxMap =
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800326 tx.getTransactionalMap(CONTINUOUS_CONSUMER_MAP, SERIALIZER);
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700327
Sho SHIMIZUfc64ffe2016-02-10 20:11:09 -0800328 for (ResourceAllocation allocation : allocations) {
329 Resource resource = allocation.resource();
330 ResourceConsumer consumer = allocation.consumer();
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700331
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800332 if (resource instanceof DiscreteResource) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800333 // if this single release fails (because the resource is allocated to another consumer,
334 // the whole release fails
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800335 if (!discreteConsumerTxMap.remove(((DiscreteResource) resource).id(), consumer)) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800336 return abortTransaction(tx);
337 }
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800338 } else if (resource instanceof ContinuousResource) {
339 ContinuousResource continuous = (ContinuousResource) resource;
Sho SHIMIZUfc64ffe2016-02-10 20:11:09 -0800340 ContinuousResourceAllocation continuousAllocation = continuousConsumerTxMap.get(continuous.id());
341 ImmutableList<ResourceAllocation> newAllocations = continuousAllocation.allocations().stream()
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800342 .filter(x -> !(x.consumer().equals(consumer) &&
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800343 ((ContinuousResource) x.resource()).value() == continuous.value()))
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800344 .collect(GuavaCollectors.toImmutableList());
345
Sho SHIMIZUfc64ffe2016-02-10 20:11:09 -0800346 if (!continuousConsumerTxMap.replace(continuous.id(), continuousAllocation,
347 new ContinuousResourceAllocation(continuousAllocation.original(), newAllocations))) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800348 return abortTransaction(tx);
349 }
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700350 }
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700351 }
Sho SHIMIZU1e0a34c2015-11-02 16:52:29 -0800352
Madan Jampani3780d4b2016-04-04 18:18:24 -0700353 return tx.commit().join() == CommitStatus.SUCCESS;
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700354 }
355
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800356 // computational complexity: O(1) if the resource is discrete type.
357 // O(n) if the resource is continuous type where n is the number of the children of
358 // the specified resource's parent
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700359 @Override
Sho SHIMIZU8fa670a2016-01-14 11:17:18 -0800360 public boolean isAvailable(Resource resource) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800361 checkNotNull(resource);
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800362 checkArgument(resource instanceof DiscreteResource || resource instanceof ContinuousResource);
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800363
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800364 if (resource instanceof DiscreteResource) {
HIGUCHI Yuta6f828c32016-01-20 18:11:05 -0800365 // check if already consumed
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800366 return getResourceAllocations(resource.id()).isEmpty();
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800367 } else {
Sho SHIMIZUf17ae282016-02-10 23:44:30 -0800368 return isAvailable((ContinuousResource) resource);
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800369 }
370 }
371
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800372 // computational complexity: O(n) where n is the number of existing allocations for the resource
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800373 private boolean isAvailable(ContinuousResource resource) {
Sho SHIMIZUf17ae282016-02-10 23:44:30 -0800374 // check if it's registered or not.
375 Versioned<Set<Resource>> children = childMap.get(resource.parent().get().id());
376 if (children == null) {
377 return false;
378 }
379
380 ContinuousResource registered = children.value().stream()
381 .filter(c -> c.id().equals(resource.id()))
382 .findFirst()
383 .map(c -> (ContinuousResource) c)
384 .get();
385 if (registered.value() < resource.value()) {
386 // Capacity < requested, can never satisfy
387 return false;
388 }
389
390 // check if there's enough left
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800391 Versioned<ContinuousResourceAllocation> allocation = continuousConsumers.get(resource.id());
392 if (allocation == null) {
HIGUCHI Yuta6f828c32016-01-20 18:11:05 -0800393 // no allocation (=no consumer) full registered resources available
394 return true;
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800395 }
396
397 return hasEnoughResource(allocation.value().original(), resource, allocation.value());
398 }
399
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800400 // computational complexity: O(n + m) where n is the number of entries in discreteConsumers
401 // and m is the number of allocations for all continuous resources
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800402 @Override
Sho SHIMIZU8fa670a2016-01-14 11:17:18 -0800403 public Collection<Resource> getResources(ResourceConsumer consumer) {
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700404 checkNotNull(consumer);
405
406 // NOTE: getting all entries may become performance bottleneck
407 // TODO: revisit for better backend data structure
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800408 Stream<DiscreteResource> discreteStream = discreteConsumers.entrySet().stream()
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700409 .filter(x -> x.getValue().value().equals(consumer))
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800410 .map(Map.Entry::getKey)
411 .map(x -> Resources.discrete(x).resource());
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800412
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800413 Stream<ContinuousResource> continuousStream = continuousConsumers.values().stream()
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800414 .flatMap(x -> x.value().allocations().stream()
415 .map(y -> Maps.immutableEntry(x.value().original(), y)))
416 .filter(x -> x.getValue().consumer().equals(consumer))
417 .map(x -> x.getKey());
418
419 return Stream.concat(discreteStream, continuousStream).collect(Collectors.toList());
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700420 }
421
Sho SHIMIZU82bfe992016-02-10 09:55:32 -0800422 // computational complexity: O(1)
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700423 @Override
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800424 public Set<Resource> getChildResources(DiscreteResourceId parent) {
Sho SHIMIZUe7f4f3f2015-10-13 16:27:25 -0700425 checkNotNull(parent);
426
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800427 Versioned<Set<Resource>> children = childMap.get(parent);
Sho SHIMIZUe7f4f3f2015-10-13 16:27:25 -0700428 if (children == null) {
Sho SHIMIZU83258ae2016-01-29 17:39:07 -0800429 return ImmutableSet.of();
Sho SHIMIZUe7f4f3f2015-10-13 16:27:25 -0700430 }
431
432 return children.value();
433 }
434
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800435 // computational complexity: O(n) where n is the number of the children of the parent
Sho SHIMIZUe7f4f3f2015-10-13 16:27:25 -0700436 @Override
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800437 public <T> Collection<Resource> getAllocatedResources(DiscreteResourceId parent, Class<T> cls) {
Sho SHIMIZU1f5e5912015-08-10 17:00:00 -0700438 checkNotNull(parent);
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700439 checkNotNull(cls);
440
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800441 Versioned<Set<Resource>> children = childMap.get(parent);
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700442 if (children == null) {
Sho SHIMIZU2c0ae122016-01-20 13:14:38 -0800443 return ImmutableList.of();
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700444 }
445
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800446 Stream<DiscreteResource> discrete = children.value().stream()
Sho SHIMIZU003ed322016-02-11 12:58:42 -0800447 .filter(x -> x.isTypeOf(cls))
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800448 .filter(x -> x instanceof DiscreteResource)
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800449 .map(x -> ((DiscreteResource) x))
450 .filter(x -> discreteConsumers.containsKey(x.id()));
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800451
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800452 Stream<ContinuousResource> continuous = children.value().stream()
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800453 .filter(x -> x.id().equals(parent.child(cls)))
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800454 .filter(x -> x instanceof ContinuousResource)
455 .map(x -> (ContinuousResource) x)
Sho SHIMIZU90039242016-02-11 09:45:32 -0800456 // we don't use cascading simple predicates like follows to reduce accesses to consistent map
457 // .filter(x -> continuousConsumers.containsKey(x.id()))
458 // .filter(x -> continuousConsumers.get(x.id()) != null)
459 // .filter(x -> !continuousConsumers.get(x.id()).value().allocations().isEmpty());
460 .filter(resource -> {
461 Versioned<ContinuousResourceAllocation> allocation = continuousConsumers.get(resource.id());
462 if (allocation == null) {
463 return false;
464 }
465 return !allocation.value().allocations().isEmpty();
466 });
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800467
468 return Stream.concat(discrete, continuous).collect(Collectors.toList());
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700469 }
Sho SHIMIZUd29847f2015-08-13 09:10:59 -0700470
471 /**
472 * Abort the transaction.
473 *
474 * @param tx transaction context
475 * @return always false
476 */
477 private boolean abortTransaction(TransactionContext tx) {
478 tx.abort();
479 return false;
480 }
481
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800482 // Appends the specified ResourceAllocation to the existing values stored in the map
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800483 // computational complexity: O(n) where n is the number of the elements in the associated allocation
484 private boolean appendValue(TransactionalMap<ContinuousResourceId, ContinuousResourceAllocation> map,
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800485 ContinuousResource original, ResourceAllocation value) {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800486 ContinuousResourceAllocation oldValue = map.putIfAbsent(original.id(),
487 new ContinuousResourceAllocation(original, ImmutableList.of(value)));
488 if (oldValue == null) {
489 return true;
490 }
491
492 if (oldValue.allocations().contains(value)) {
493 // don't write to map because all values are already stored
494 return true;
495 }
496
497 ContinuousResourceAllocation newValue = new ContinuousResourceAllocation(original,
498 ImmutableList.<ResourceAllocation>builder()
499 .addAll(oldValue.allocations())
500 .add(value)
501 .build());
502 return map.replace(original.id(), oldValue, newValue);
503 }
Sho SHIMIZUd29847f2015-08-13 09:10:59 -0700504 /**
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700505 * Appends the values to the existing values associated with the specified key.
Sho SHIMIZU4568c412015-08-21 16:39:07 -0700506 * If the map already has all the given values, appending will not happen.
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700507 *
508 * @param map map holding multiple values for a key
509 * @param key key specifying values
510 * @param values values to be appended
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700511 * @return true if the operation succeeds, false otherwise.
512 */
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800513 // computational complexity: O(n) where n is the number of the specified value
514 private boolean appendValues(TransactionalMap<DiscreteResourceId, Set<Resource>> map,
515 DiscreteResourceId key, List<Resource> values) {
Sho SHIMIZU67c90102016-02-23 10:49:58 -0800516 Set<Resource> requested = new LinkedHashSet<>(values);
517 Set<Resource> oldValues = map.putIfAbsent(key, requested);
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700518 if (oldValues == null) {
Sho SHIMIZU93a74b32015-11-09 11:48:23 -0800519 return true;
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700520 }
521
Sho SHIMIZU67c90102016-02-23 10:49:58 -0800522 Set<Resource> addedValues = Sets.difference(requested, oldValues);
523 // no new value, then no-op
Sho SHIMIZU1992daf2016-02-17 17:38:55 -0800524 if (addedValues.isEmpty()) {
Sho SHIMIZU4568c412015-08-21 16:39:07 -0700525 // don't write to map because all values are already stored
526 return true;
527 }
528
Sho SHIMIZU67c90102016-02-23 10:49:58 -0800529 Set<ResourceId> addedIds = addedValues.stream()
530 .map(Resource::id)
531 .collect(Collectors.toSet());
532 // if the value is not found but the same ID is found
533 // (this happens only when being continuous resource)
534 if (oldValues.stream().anyMatch(x -> addedIds.contains(x.id()))) {
535 // no-op, but indicating failure (reject the request)
536 return false;
537 }
538 Set<Resource> newValues = new LinkedHashSet<>(oldValues);
Sho SHIMIZU1992daf2016-02-17 17:38:55 -0800539 newValues.addAll(addedValues);
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800540 return map.replace(key, oldValues, newValues);
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700541 }
542
543 /**
Sho SHIMIZUba1f83b2015-10-14 08:11:20 -0700544 * Removes the values from the existing values associated with the specified key.
Sho SHIMIZU5618ee52015-08-21 17:19:44 -0700545 * If the map doesn't contain the given values, removal will not happen.
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700546 *
547 * @param map map holding multiple values for a key
548 * @param key key specifying values
549 * @param values values to be removed
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700550 * @return true if the operation succeeds, false otherwise
551 */
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800552 // computational complexity: O(n) where n is the number of the specified values
553 private boolean removeValues(TransactionalMap<DiscreteResourceId, Set<Resource>> map,
554 DiscreteResourceId key, List<Resource> values) {
Sho SHIMIZU07b7bc92016-01-29 18:27:58 -0800555 Set<Resource> oldValues = map.putIfAbsent(key, new LinkedHashSet<>());
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700556 if (oldValues == null) {
HIGUCHI Yutadc4394c2016-01-29 15:35:10 -0800557 log.trace("No-Op removing values. key {} did not exist", key);
Sho SHIMIZU93a74b32015-11-09 11:48:23 -0800558 return true;
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700559 }
560
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800561 if (values.stream().allMatch(x -> !oldValues.contains(x))) {
Sho SHIMIZU5618ee52015-08-21 17:19:44 -0700562 // don't write map because none of the values are stored
HIGUCHI Yutadc4394c2016-01-29 15:35:10 -0800563 log.trace("No-Op removing values. key {} did not contain {}", key, values);
Sho SHIMIZU5618ee52015-08-21 17:19:44 -0700564 return true;
565 }
566
Sho SHIMIZU07b7bc92016-01-29 18:27:58 -0800567 LinkedHashSet<Resource> newValues = new LinkedHashSet<>(oldValues);
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800568 newValues.removeAll(values);
569 return map.replace(key, oldValues, newValues);
Sho SHIMIZU2d8a13a2015-08-18 22:37:41 -0700570 }
571
572 /**
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800573 * Returns the resource which has the same key as the specified resource ID
574 * in the set as a value of the map.
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700575 *
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800576 * @param childTxMap map storing parent - child relationship of resources
577 * @param id ID of resource to be checked
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800578 * @return the resource which is regarded as the same as the specified resource
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700579 */
Sho SHIMIZUa6a6fd32016-02-10 18:36:44 -0800580 // Naive implementation, which traverses all elements in the set when continuous resource
581 // computational complexity: O(1) when discrete resource. O(n) when continuous resource
582 // where n is the number of elements in the associated set
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800583 private Optional<Resource> lookup(TransactionalMap<DiscreteResourceId, Set<Resource>> childTxMap, ResourceId id) {
584 if (!id.parent().isPresent()) {
585 return Optional.of(Resource.ROOT);
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700586 }
587
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800588 Set<Resource> values = childTxMap.get(id.parent().get());
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800589 if (values == null) {
590 return Optional.empty();
591 }
592
Sho SHIMIZUa6a6fd32016-02-10 18:36:44 -0800593 // short-circuit if discrete resource
594 // check the existence in the set: O(1) operation
595 if (id instanceof DiscreteResourceId) {
596 DiscreteResource discrete = Resources.discrete((DiscreteResourceId) id).resource();
597 if (values.contains(discrete)) {
598 return Optional.of(discrete);
599 } else {
600 return Optional.empty();
601 }
602 }
603
604 // continuous resource case
605 // iterate over the values in the set: O(n) operation
Sho SHIMIZU72f81b12016-02-09 09:26:17 -0800606 return values.stream()
607 .filter(x -> x.id().equals(id))
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800608 .findFirst();
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800609 }
610
611 /**
612 * Checks if there is enough resource volume to allocated the requested resource
613 * against the specified resource.
614 *
615 * @param original original resource
616 * @param request requested resource
617 * @param allocation current allocation of the resource
618 * @return true if there is enough resource volume. Otherwise, false.
619 */
Sho SHIMIZUdd3750c2016-02-01 11:37:04 -0800620 // computational complexity: O(n) where n is the number of allocations
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800621 private boolean hasEnoughResource(ContinuousResource original,
622 ContinuousResource request,
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800623 ContinuousResourceAllocation allocation) {
624 if (allocation == null) {
625 return request.value() <= original.value();
626 }
627
628 double allocated = allocation.allocations().stream()
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800629 .filter(x -> x.resource() instanceof ContinuousResource)
630 .map(x -> (ContinuousResource) x.resource())
631 .mapToDouble(ContinuousResource::value)
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800632 .sum();
633 double left = original.value() - allocated;
634 return request.value() <= left;
635 }
636
637 // internal use only
638 private static final class ContinuousResourceAllocation {
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800639 private final ContinuousResource original;
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800640 private final ImmutableList<ResourceAllocation> allocations;
641
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800642 private ContinuousResourceAllocation(ContinuousResource original,
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800643 ImmutableList<ResourceAllocation> allocations) {
644 this.original = original;
645 this.allocations = allocations;
646 }
647
Sho SHIMIZUf33b8932016-01-25 18:43:32 -0800648 private ContinuousResource original() {
Sho SHIMIZU6c9e33a2016-01-07 18:45:27 -0800649 return original;
650 }
651
652 private ImmutableList<ResourceAllocation> allocations() {
653 return allocations;
654 }
Sho SHIMIZUba41fc12015-08-12 15:43:22 -0700655 }
Sho SHIMIZU78ee25c2015-07-16 15:54:14 -0700656}