Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2017-present Open Networking Foundation |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package org.onosproject.pipelines.fabric.pipeliner; |
| 18 | |
| 19 | import com.google.common.collect.ImmutableList; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 20 | import org.onlab.util.KryoNamespace; |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 21 | import org.onlab.util.SharedExecutors; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 22 | import org.onosproject.net.DeviceId; |
| 23 | import org.onosproject.net.PortNumber; |
| 24 | import org.onosproject.net.behaviour.NextGroup; |
| 25 | import org.onosproject.net.behaviour.Pipeliner; |
| 26 | import org.onosproject.net.behaviour.PipelinerContext; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 27 | import org.onosproject.net.flow.FlowRule; |
| 28 | import org.onosproject.net.flow.FlowRuleOperations; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 29 | import org.onosproject.net.flow.FlowRuleService; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 30 | import org.onosproject.net.flowobjective.FilteringObjective; |
| 31 | import org.onosproject.net.flowobjective.FlowObjectiveStore; |
| 32 | import org.onosproject.net.flowobjective.ForwardingObjective; |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 33 | import org.onosproject.net.flowobjective.IdNextTreatment; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 34 | import org.onosproject.net.flowobjective.NextObjective; |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 35 | import org.onosproject.net.flowobjective.NextTreatment; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 36 | import org.onosproject.net.flowobjective.Objective; |
| 37 | import org.onosproject.net.flowobjective.ObjectiveError; |
| 38 | import org.onosproject.net.group.GroupDescription; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 39 | import org.onosproject.net.group.GroupService; |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 40 | import org.onosproject.pipelines.fabric.AbstractFabricHandlerBehavior; |
Daniele Moro | f51d0c1 | 2019-07-30 10:43:10 -0700 | [diff] [blame^] | 41 | import org.onosproject.pipelines.fabric.FabricCapabilities; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 42 | import org.onosproject.store.serializers.KryoNamespaces; |
| 43 | import org.slf4j.Logger; |
| 44 | |
| 45 | import java.util.Collection; |
| 46 | import java.util.List; |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 47 | import java.util.Objects; |
Yi Tseng | fe13f3e | 2018-08-19 03:09:54 +0800 | [diff] [blame] | 48 | import java.util.concurrent.CompletableFuture; |
Yi Tseng | fe13f3e | 2018-08-19 03:09:54 +0800 | [diff] [blame] | 49 | import java.util.concurrent.ExecutorService; |
Yi Tseng | 1b154bd | 2017-11-20 17:48:19 -0800 | [diff] [blame] | 50 | import java.util.stream.Collectors; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 51 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 52 | import static java.lang.String.format; |
| 53 | import static org.onosproject.pipelines.fabric.FabricUtils.outputPort; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 54 | import static org.slf4j.LoggerFactory.getLogger; |
| 55 | |
| 56 | /** |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 57 | * Pipeliner implementation for fabric pipeline which uses ObjectiveTranslator |
| 58 | * implementations to translate flow objectives for the different blocks, |
| 59 | * filtering, forwarding and next. |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 60 | */ |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 61 | public class FabricPipeliner extends AbstractFabricHandlerBehavior |
| 62 | implements Pipeliner { |
| 63 | |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 64 | private static final Logger log = getLogger(FabricPipeliner.class); |
| 65 | |
| 66 | protected static final KryoNamespace KRYO = new KryoNamespace.Builder() |
| 67 | .register(KryoNamespaces.API) |
| 68 | .register(FabricNextGroup.class) |
| 69 | .build("FabricPipeliner"); |
| 70 | |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 71 | protected DeviceId deviceId; |
| 72 | protected FlowRuleService flowRuleService; |
| 73 | protected GroupService groupService; |
| 74 | protected FlowObjectiveStore flowObjectiveStore; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 75 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 76 | private FilteringObjectiveTranslator filteringTranslator; |
| 77 | private ForwardingObjectiveTranslator forwardingTranslator; |
| 78 | private NextObjectiveTranslator nextTranslator; |
Charles Chan | 91ea972 | 2018-08-30 15:56:32 -0700 | [diff] [blame] | 79 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 80 | private final ExecutorService callbackExecutor = SharedExecutors.getPoolThreadExecutor(); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 81 | |
Daniele Moro | f51d0c1 | 2019-07-30 10:43:10 -0700 | [diff] [blame^] | 82 | /** |
| 83 | * Creates a new instance of this behavior with the given capabilities. |
| 84 | * |
| 85 | * @param capabilities capabilities |
| 86 | */ |
| 87 | public FabricPipeliner(FabricCapabilities capabilities) { |
| 88 | super(capabilities); |
| 89 | } |
| 90 | |
| 91 | /** |
| 92 | * Create a new instance of this behaviour. Used by the abstract projectable |
| 93 | * model (i.e., {@link org.onosproject.net.Device#as(Class)}. |
| 94 | */ |
| 95 | public FabricPipeliner() { |
| 96 | super(); |
| 97 | } |
| 98 | |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 99 | @Override |
| 100 | public void init(DeviceId deviceId, PipelinerContext context) { |
| 101 | this.deviceId = deviceId; |
| 102 | this.flowRuleService = context.directory().get(FlowRuleService.class); |
| 103 | this.groupService = context.directory().get(GroupService.class); |
| 104 | this.flowObjectiveStore = context.directory().get(FlowObjectiveStore.class); |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 105 | this.filteringTranslator = new FilteringObjectiveTranslator(deviceId, capabilities); |
| 106 | this.forwardingTranslator = new ForwardingObjectiveTranslator(deviceId, capabilities); |
| 107 | this.nextTranslator = new NextObjectiveTranslator(deviceId, capabilities); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 108 | } |
| 109 | |
| 110 | @Override |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 111 | public void filter(FilteringObjective obj) { |
| 112 | final ObjectiveTranslation result = filteringTranslator.translate(obj); |
| 113 | handleResult(obj, result); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 114 | } |
| 115 | |
| 116 | @Override |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 117 | public void forward(ForwardingObjective obj) { |
| 118 | final ObjectiveTranslation result = forwardingTranslator.translate(obj); |
| 119 | handleResult(obj, result); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 120 | } |
| 121 | |
| 122 | @Override |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 123 | public void next(NextObjective obj) { |
| 124 | if (obj.op() == Objective.Operation.VERIFY) { |
Yi Tseng | 1b154bd | 2017-11-20 17:48:19 -0800 | [diff] [blame] | 125 | // TODO: support VERIFY operation |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 126 | log.debug("VERIFY operation not yet supported for NextObjective, will return success"); |
| 127 | success(obj); |
Yi Tseng | 1b154bd | 2017-11-20 17:48:19 -0800 | [diff] [blame] | 128 | return; |
| 129 | } |
| 130 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 131 | if (obj.op() == Objective.Operation.MODIFY) { |
Charles Chan | 91ea972 | 2018-08-30 15:56:32 -0700 | [diff] [blame] | 132 | // TODO: support MODIFY operation |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 133 | log.warn("MODIFY operation not yet supported for NextObjective, will return failure :("); |
| 134 | fail(obj, ObjectiveError.UNSUPPORTED); |
Charles Chan | 91ea972 | 2018-08-30 15:56:32 -0700 | [diff] [blame] | 135 | return; |
| 136 | } |
| 137 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 138 | final ObjectiveTranslation result = nextTranslator.translate(obj); |
| 139 | handleResult(obj, result); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 140 | } |
| 141 | |
| 142 | @Override |
| 143 | public List<String> getNextMappings(NextGroup nextGroup) { |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 144 | final FabricNextGroup fabricNextGroup = KRYO.deserialize(nextGroup.data()); |
| 145 | return fabricNextGroup.nextMappings().stream() |
| 146 | .map(m -> format("%s -> %s", fabricNextGroup.type(), m)) |
Yi Tseng | 1b154bd | 2017-11-20 17:48:19 -0800 | [diff] [blame] | 147 | .collect(Collectors.toList()); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 148 | } |
| 149 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 150 | private void handleResult(Objective obj, ObjectiveTranslation result) { |
| 151 | if (result.error().isPresent()) { |
| 152 | fail(obj, result.error().get()); |
| 153 | return; |
| 154 | } |
| 155 | processGroups(obj, result.groups()); |
| 156 | processFlows(obj, result.flowRules()); |
| 157 | if (obj instanceof NextObjective) { |
| 158 | handleNextGroup((NextObjective) obj); |
| 159 | } |
| 160 | success(obj); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 161 | } |
| 162 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 163 | private void handleNextGroup(NextObjective obj) { |
| 164 | switch (obj.op()) { |
| 165 | case REMOVE: |
| 166 | removeNextGroup(obj); |
| 167 | break; |
| 168 | case ADD: |
| 169 | case ADD_TO_EXISTING: |
| 170 | case REMOVE_FROM_EXISTING: |
| 171 | case MODIFY: |
| 172 | putNextGroup(obj); |
| 173 | break; |
| 174 | case VERIFY: |
| 175 | break; |
| 176 | default: |
| 177 | log.error("Unknown NextObjective operation '{}'", obj.op()); |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | private void processFlows(Objective objective, Collection<FlowRule> flowRules) { |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 182 | if (flowRules.isEmpty()) { |
Yi Tseng | f78e174 | 2018-04-08 19:57:17 +0800 | [diff] [blame] | 183 | return; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 184 | } |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 185 | final FlowRuleOperations.Builder ops = FlowRuleOperations.builder(); |
| 186 | switch (objective.op()) { |
| 187 | case ADD: |
| 188 | case ADD_TO_EXISTING: |
| 189 | flowRules.forEach(ops::add); |
| 190 | break; |
| 191 | case REMOVE: |
| 192 | case REMOVE_FROM_EXISTING: |
| 193 | flowRules.forEach(ops::remove); |
| 194 | break; |
| 195 | default: |
| 196 | log.warn("Unsupported Objective operation '{}'", objective.op()); |
| 197 | return; |
wu | 914ed23 | 2018-10-23 11:19:53 +0800 | [diff] [blame] | 198 | } |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 199 | flowRuleService.apply(ops.build()); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 200 | } |
| 201 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 202 | private void processGroups(Objective objective, Collection<GroupDescription> groups) { |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 203 | if (groups.isEmpty()) { |
Yi Tseng | f78e174 | 2018-04-08 19:57:17 +0800 | [diff] [blame] | 204 | return; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 205 | } |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 206 | switch (objective.op()) { |
| 207 | case ADD: |
| 208 | groups.forEach(groupService::addGroup); |
| 209 | break; |
| 210 | case REMOVE: |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 211 | groups.forEach(group -> groupService.removeGroup( |
| 212 | deviceId, group.appCookie(), objective.appId())); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 213 | break; |
Yi Tseng | 1b154bd | 2017-11-20 17:48:19 -0800 | [diff] [blame] | 214 | case ADD_TO_EXISTING: |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 215 | groups.forEach(group -> groupService.addBucketsToGroup( |
| 216 | deviceId, group.appCookie(), group.buckets(), |
| 217 | group.appCookie(), group.appId()) |
Charles Chan | 91ea972 | 2018-08-30 15:56:32 -0700 | [diff] [blame] | 218 | ); |
Yi Tseng | 1b154bd | 2017-11-20 17:48:19 -0800 | [diff] [blame] | 219 | break; |
| 220 | case REMOVE_FROM_EXISTING: |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 221 | groups.forEach(group -> groupService.removeBucketsFromGroup( |
| 222 | deviceId, group.appCookie(), group.buckets(), |
| 223 | group.appCookie(), group.appId()) |
Charles Chan | 91ea972 | 2018-08-30 15:56:32 -0700 | [diff] [blame] | 224 | ); |
Yi Tseng | 1b154bd | 2017-11-20 17:48:19 -0800 | [diff] [blame] | 225 | break; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 226 | default: |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 227 | log.warn("Unsupported Objective operation {}", objective.op()); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 228 | } |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 229 | } |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 230 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 231 | private void fail(Objective objective, ObjectiveError error) { |
| 232 | CompletableFuture.runAsync( |
| 233 | () -> objective.context().ifPresent( |
| 234 | ctx -> ctx.onError(objective, error)), callbackExecutor); |
Yi Tseng | fe13f3e | 2018-08-19 03:09:54 +0800 | [diff] [blame] | 235 | |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 236 | } |
| 237 | |
Charles Chan | 91ea972 | 2018-08-30 15:56:32 -0700 | [diff] [blame] | 238 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 239 | private void success(Objective objective) { |
| 240 | CompletableFuture.runAsync( |
| 241 | () -> objective.context().ifPresent( |
| 242 | ctx -> ctx.onSuccess(objective)), callbackExecutor); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 243 | } |
| 244 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 245 | private void removeNextGroup(NextObjective obj) { |
| 246 | final NextGroup removed = flowObjectiveStore.removeNextGroup(obj.id()); |
| 247 | if (removed == null) { |
| 248 | log.debug("NextGroup {} was not found in FlowObjectiveStore"); |
| 249 | } |
Charles Chan | 91ea972 | 2018-08-30 15:56:32 -0700 | [diff] [blame] | 250 | } |
| 251 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 252 | private void putNextGroup(NextObjective obj) { |
| 253 | final List<String> nextMappings = obj.nextTreatments().stream() |
| 254 | .map(this::nextTreatmentToMappingString) |
| 255 | .filter(Objects::nonNull) |
| 256 | .collect(Collectors.toList()); |
| 257 | final FabricNextGroup nextGroup = new FabricNextGroup(obj.type(), nextMappings); |
| 258 | flowObjectiveStore.putNextGroup(obj.id(), nextGroup); |
| 259 | } |
| 260 | |
| 261 | private String nextTreatmentToMappingString(NextTreatment n) { |
| 262 | switch (n.type()) { |
| 263 | case TREATMENT: |
| 264 | final PortNumber p = outputPort(n); |
| 265 | return p == null ? "UNKNOWN" |
| 266 | : format("OUTPUT:%s", p.toString()); |
| 267 | case ID: |
| 268 | final IdNextTreatment id = (IdNextTreatment) n; |
| 269 | return format("NEXT_ID:%d", id.nextId()); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 270 | default: |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 271 | log.warn("Unknown NextTreatment type '{}'", n.type()); |
| 272 | return "???"; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 273 | } |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 274 | } |
| 275 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 276 | /** |
| 277 | * NextGroup implementation. |
| 278 | */ |
| 279 | private static class FabricNextGroup implements NextGroup { |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 280 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 281 | private final NextObjective.Type type; |
| 282 | private final List<String> nextMappings; |
| 283 | |
| 284 | FabricNextGroup(NextObjective.Type type, List<String> nextMappings) { |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 285 | this.type = type; |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 286 | this.nextMappings = ImmutableList.copyOf(nextMappings); |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 287 | } |
| 288 | |
Charles Chan | 91ea972 | 2018-08-30 15:56:32 -0700 | [diff] [blame] | 289 | NextObjective.Type type() { |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 290 | return type; |
| 291 | } |
| 292 | |
Carmelo Cascone | b5324e7 | 2018-11-25 02:26:32 -0800 | [diff] [blame] | 293 | Collection<String> nextMappings() { |
| 294 | return nextMappings; |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | @Override |
| 298 | public byte[] data() { |
| 299 | return KRYO.serialize(this); |
| 300 | } |
| 301 | } |
Yi Tseng | 0b80972 | 2017-11-03 10:23:26 -0700 | [diff] [blame] | 302 | } |