Refactored INT service impl to support multi-instance ONOS and fabric.p4
Change-Id: Ic82a3ab72d71a774606b25997e283b93aedc6ec9
diff --git a/apps/inbandtelemetry/BUILD b/apps/inbandtelemetry/BUILD
new file mode 100644
index 0000000..3a6d7eb
--- /dev/null
+++ b/apps/inbandtelemetry/BUILD
@@ -0,0 +1,15 @@
+BUNDLES = [
+ "//apps/inbandtelemetry/api:onos-apps-inbandtelemetry-api",
+ "//apps/inbandtelemetry/impl:onos-apps-inbandtelemetry-impl",
+ "//apps/inbandtelemetry/app:onos-apps-inbandtelemetry-app",
+]
+
+onos_app(
+ app_name = "org.onosproject.inbandtelemetry",
+ category = "Monitoring",
+ description = "Provides managements of INT-capable devices. Specifies flows to enable INT and" +
+ "types of metadata to collect. Sets up INT-related information.",
+ included_bundles = BUNDLES,
+ title = "P4 In-band Network Telemetry Service",
+ url = "http://onosproject.org",
+)
diff --git a/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntConfig.java b/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntConfig.java
index a3e6148..c5af580 100644
--- a/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntConfig.java
+++ b/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntConfig.java
@@ -30,7 +30,7 @@
/**
* Represents a type of telemetry spec to collect in the dataplane.
*/
- enum TelemetrySpec {
+ public enum TelemetrySpec {
/**
* Embeds telemetry metadata according to the INT specification.
*
diff --git a/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntProgrammable.java b/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntProgrammable.java
index 452b787..507f870 100644
--- a/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntProgrammable.java
+++ b/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntProgrammable.java
@@ -16,18 +16,64 @@
package org.onosproject.inbandtelemetry.api;
import com.google.common.annotations.Beta;
+import org.onosproject.net.PortNumber;
import org.onosproject.net.driver.HandlerBehaviour;
-import java.util.concurrent.CompletableFuture;
-
+/**
+ * Abstraction of a device implementing In-band Network Telemetry (INT)
+ * capabilities.
+ */
@Beta
public interface IntProgrammable extends HandlerBehaviour {
/**
- * Initializes the pipeline, by installing required flow rules
- * not relevant to specific watchlist, report and event.
+ * INT functionalities that a device can implement.
*/
- void init();
+ enum IntFunctionality {
+ /**
+ * Source functionality.
+ */
+ SOURCE,
+ /**
+ * Sink functionality.
+ */
+ SINK,
+ /**
+ * Transit functionality.
+ */
+ TRANSIT
+ }
+
+ /**
+ * Initializes the pipeline, by installing required flow rules not relevant
+ * to specific watchlist, report and event. Returns true if the operation
+ * was successful, false otherwise.
+ *
+ * @return true if successful, false otherwise
+ */
+ boolean init();
+
+ /**
+ * Configures the given port as an INT source port. Packets received via
+ * this port can be modified to add the INT header, if a corresponding INT
+ * objective is matched. Returns true if the operation was successful, false
+ * otherwise.
+ *
+ * @param port port
+ * @return true if successful, false otherwise
+ */
+ boolean setSourcePort(PortNumber port);
+
+ /**
+ * Configures the given port as an INT sink port. Packets forwarded via this
+ * port will be stripped of the INT header and a corresponding INT report
+ * will be generated. Returns true if the operation was successful, false
+ * otherwise.
+ *
+ * @param port port
+ * @return true if successful, false otherwise
+ */
+ boolean setSinkPort(PortNumber port);
/**
* Adds a given IntObjective to the device.
@@ -35,7 +81,7 @@
* @param obj an IntObjective
* @return true if the objective is successfully added; false otherwise.
*/
- CompletableFuture<Boolean> addIntObjective(IntObjective obj);
+ boolean addIntObjective(IntObjective obj);
/**
* Removes a given IntObjective entry from the device.
@@ -43,7 +89,7 @@
* @param obj an IntObjective
* @return true if the objective is successfully removed; false otherwise.
*/
- CompletableFuture<Boolean> removeIntObjective(IntObjective obj);
+ boolean removeIntObjective(IntObjective obj);
/**
* Set up report-related configuration.
@@ -51,7 +97,20 @@
* @param config a configuration regarding to the collector
* @return true if the objective is successfully added; false otherwise.
*/
- CompletableFuture<Boolean> setupIntConfig(IntConfig config);
+ boolean setupIntConfig(IntConfig config);
+
+ /**
+ * Clean up any INT-related configuration from the device.
+ */
+ void cleanup();
+
+ /**
+ * Returns true if this device supports the given INT functionality.
+ *
+ * @param functionality INt functionality
+ * @return true if functionality is supported, false otherwise
+ */
+ boolean supportsFunctionality(IntFunctionality functionality);
//TODO: [ONOS-7616] Design IntEvent and related APIs
}
diff --git a/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntService.java b/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntService.java
index ef2e1ad..59e65c3 100644
--- a/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntService.java
+++ b/apps/inbandtelemetry/api/src/main/java/org/onosproject/inbandtelemetry/api/IntService.java
@@ -43,13 +43,13 @@
}
/**
- * Starts the INT functionalities in all INT-capable devices.
+ * Starts the INT functionality in all INT-capable devices.
* This will include populating tables to process INT packets.
*/
void startInt();
/**
- * Starts the INT functionalities in specified set of INT transit devices.
+ * Starts the INT functionality in specified set of INT transit devices.
* <p>
* Note: this is an experimental API, which can be either changed or removed.
*
@@ -116,4 +116,4 @@
Map<IntIntentId, IntIntent> getIntIntents();
//TODO: [ONOS-7616] Design IntEvent and related APIs
-}
\ No newline at end of file
+}
diff --git a/apps/inbandtelemetry/app/BUILD b/apps/inbandtelemetry/app/BUILD
new file mode 100644
index 0000000..3391ffa
--- /dev/null
+++ b/apps/inbandtelemetry/app/BUILD
@@ -0,0 +1,11 @@
+COMPILE_DEPS = CORE_DEPS + KRYO + JACKSON + [
+ "//core/store/serializers:onos-core-serializers",
+ "//apps/inbandtelemetry/api:onos-apps-inbandtelemetry-api",
+]
+
+TEST_DEPS = TEST_REST
+
+osgi_jar_with_tests(
+ test_deps = TEST_DEPS,
+ deps = COMPILE_DEPS,
+)
diff --git a/apps/inbandtelemetry/app/src/main/java/org/onosproject/inbandtelemetry/app/IntControl.java b/apps/inbandtelemetry/app/src/main/java/org/onosproject/inbandtelemetry/app/IntControl.java
deleted file mode 100644
index efd3590..0000000
--- a/apps/inbandtelemetry/app/src/main/java/org/onosproject/inbandtelemetry/app/IntControl.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2015-present Open Networking Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.onosproject.inbandtelemetry.app;
-
-import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Deactivate;
-import org.apache.felix.scr.annotations.Reference;
-import org.apache.felix.scr.annotations.Component;
-import org.apache.felix.scr.annotations.ReferenceCardinality;
-import org.onosproject.core.CoreService;
-import org.onosproject.inbandtelemetry.api.IntService;
-import org.onosproject.net.host.HostService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Component(immediate = true)
-public class IntControl {
- private final Logger log = LoggerFactory.getLogger(getClass());
-// private ApplicationId appId;
-// private static final int collectorPort = 1234;
-// private static final IpAddress collectorIp = IpAddress.valueOf("10.0.0.3");
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected CoreService coreService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected IntService intService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected HostService hostService;
-
- @Activate
- protected void activate() {
- coreService.registerApplication("org.onosproject.inbandtelemetry.app");
- log.info("Started");
- }
-
- @Deactivate
- protected void deactivate() {
- log.info("Stopped");
- }
-}
diff --git a/apps/inbandtelemetry/app/src/main/java/org/onosproject/inbandtelemetry/app/package-info.java b/apps/inbandtelemetry/app/src/main/java/org/onosproject/inbandtelemetry/app/package-info.java
deleted file mode 100644
index 859bdf7..0000000
--- a/apps/inbandtelemetry/app/src/main/java/org/onosproject/inbandtelemetry/app/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright 2015-present Open Networking Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * IntService sample application.
- */
-package org.onosproject.inbandtelemetry.app;
diff --git a/apps/inbandtelemetry/app/src/main/resources/app/view/intApp/intApp.css b/apps/inbandtelemetry/app/src/main/resources/app/view/intApp/intApp.css
index 1cfd447..7c2bbe2 100644
--- a/apps/inbandtelemetry/app/src/main/resources/app/view/intApp/intApp.css
+++ b/apps/inbandtelemetry/app/src/main/resources/app/view/intApp/intApp.css
@@ -58,6 +58,10 @@
text-align: center;
}
+#ov-int-app-main input {
+ padding: 4px;
+ font-size: inherit;
+}
.light #ov-int-app-main .int-app-config-button {
color: white;
@@ -68,8 +72,25 @@
background-color: #aaa;
}
/*---------------------------------------------------------------------------*/
+#ov-int-app-main hr {
+ border: 0;
+ height: 1px;
+ background: #333;
+}
+
#ov-int-app-main h2 {
display: inline-block;
+ margin: 15px 0px 15px;
+}
+
+#ov-int-app-main h3 {
+ display: inline-block;
+ margin-bottom: 10px;
+}
+
+#ov-int-app-main h4 {
+ display: inline-block;
+ margin-bottom: 10px;
}
/* #ov-int-app-main .table-body{
display: inline-block;
diff --git a/apps/inbandtelemetry/app/src/main/resources/app/view/intApp/intApp.html b/apps/inbandtelemetry/app/src/main/resources/app/view/intApp/intApp.html
index 42717e0..8c39cdc 100644
--- a/apps/inbandtelemetry/app/src/main/resources/app/view/intApp/intApp.html
+++ b/apps/inbandtelemetry/app/src/main/resources/app/view/intApp/intApp.html
@@ -1,62 +1,128 @@
<!-- partial HTML -->
<div id="ov-int-app-main">
- <div class="config-panel">
- <div>
- Collector IP
- <input type="text" required pattern="^([0-9]{1,3}\.){3}[0-9]{1,3}$" ng-model="collectorIp">
+ <div class="tabular-header">
+ <h2 style="font-weight: bold">In-band Network Telemetry (INT) Control
+ Application</h2>
+ </div>
- Collector Port
- <input type="text" required pattern="^[0-9]{0,5}$" ng-model="collectorPort">
- </div>
- </div>
- <div class="config-button-panel">
- <div class = "int-app-config-button" ng-click="sendIntConfigString()">
- Deploy
- </div>
- </div>
<hr>
- <div class="input-panel">
- <div>
- Src Address
- <input type="text" required pattern="^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$" ng-model="ip4SrcPrefix">
- Dst Address
- <input type="text" required pattern="^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$" ng-model="ip4DstPrefix">
-
- Src Port
- <input type="text" required pattern="^[0-9]{0,5}$" ng-model="l4SrcPort">
-
- Dst Port
- <input type="text" required pattern="^[0-9]{0,5}$" ng-model="l4DstPort">
-
- Protocol
- <select name="protocol" ng-model="protocol">
- <option selected disabled hidden style="display: none" value=''></option>
- <option value="TCP">TCP</option>
- <option value="UDP">UDP</option>
- </select>
+ <div class="tabular-header">
+ <h2>
+ INT Collector Configuration
+ </h2>
+ <div class="config-panel">
+ <h3>
+ Collector IPv4 address and UDP port
+ </h3>
+ <div>
+ <input type="text" required
+ pattern="^([0-9]{1,3}\.){3}[0-9]{1,3}$"
+ ng-model="collectorIp" placeholder="IPv4 address">
+ :
+ <input type="text" required pattern="^[0-9]{0,5}$"
+ ng-model="collectorPort" placeholder="port">
+ </div>
</div>
- <div>
- <input type="checkbox" ng-model= "metaSwId">Switch Id
- <input type="checkbox" ng-model= "metaPortId">Port Id
- <input type="checkbox" ng-model= "metaHopLatency">Hop Latency
- <input type="checkbox" ng-model= "metaQOccupancy">Queue Occupancy
- <input type="checkbox" ng-model= "metaIngressTstamp">Ingress Timestamp
- <input type="checkbox" ng-model= "metaEgressTstamp">Egress Timestamp
- <!--<input type="checkbox" ng-model= "metaQCongestion">Queue Congestion Status-->
- <input type="checkbox" ng-model= "metaEgressTx">Egress Port Tx Utilization
+ <div class="config-button-panel">
+ <div class="int-app-config-button" ng-click="sendIntConfigString()">
+ Apply Configuration
+ </div>
</div>
</div>
- <div class="button-panel">
- <div class = "int-app-button" ng-click="sendIntIntentString()">
- Deploy
+ <hr>
+
+ <div class="tabular-header">
+ <h2>
+ INT Watchlist Rules
+ </h2>
+
+ <div>
+ <div class="input-panel">
+ <h3>
+ Create New Watchlist Rule
+ </h3>
+ <div>
+ <label>
+ <input placeholder="Source IP address" type="text"
+ required
+ pattern="^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$"
+ ng-model="ip4SrcPrefix">
+ </label>
+ <label>
+ <input placeholder="Dest IP address" type="text"
+ required
+ pattern="^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$"
+ ng-model="ip4DstPrefix">
+ </label>
+ <label>
+ <input placeholder="Source port" type="text" required
+ pattern="^[0-9]{0,5}$"
+ ng-model="l4SrcPort">
+ </label>
+ <label>
+ <input placeholder="Dest port" type="text" required
+ pattern="^[0-9]{0,5}$"
+ ng-model="l4DstPort">
+ </label>
+ <label>
+ Protocol
+ <select name="protocol" ng-model="protocol">
+ <option selected disabled hidden
+ style="display: none" value=''></option>
+ <option value="TCP">TCP</option>
+ <option value="UDP">UDP</option>
+ </select>
+ </label>
+ </div>
+ <div>
+ <label>
+ <input type="checkbox" ng-model="metaSwId">
+ Switch ID
+ </label>
+ <label>
+ <input type="checkbox" ng-model="metaPortId">
+ Port IDs
+ </label>
+ <label>
+ <input type="checkbox" ng-model="metaHopLatency">
+ Hop Latency
+ </label>
+ <label>
+ <input type="checkbox" ng-model="metaQOccupancy">
+ Queue Occupancy
+ </label>
+ <label>
+ <input type="checkbox" ng-model="metaIngressTstamp">
+ Ingress Timestamp
+ </label>
+ <label>
+ <input type="checkbox" ng-model="metaEgressTstamp">
+ Egress Timestamp
+ </label>
+ <label>
+ <input type="checkbox" ng-model="metaQCongestion">
+ Queue Congestion Status
+ </label>
+ <label>
+ <input type="checkbox" ng-model="metaEgressTx">
+ Egress Port Tx Utilization
+ </label>
+ </div>
+ </div>
+ </div>
+
+ <div class="button-panel">
+ <div class="int-app-button" ng-click="sendIntIntentString()">
+ Apply Watchlist Rule
+ </div>
</div>
</div>
<!-- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<div class='int-app-main-intents'>
<div class="tabular-header">
- <h2>Installed INT Intents ({{tableData.length}} total)</h2>
+ <h2>Installed Watchlist Rules ({{tableData.length}} total)</h2>
<div class="ctrl-btns">
<div class="refresh" ng-class="{active: autoRefresh}"
icon icon-id="refresh" icon-size="36"
@@ -77,13 +143,13 @@
<div class="table-header" onos-sortable-header>
<table>
<tr>
- <td colId="id" sortable>ID </td>
- <td colId="srcAddr" sortable>Src Address </td>
- <td colId="dstAddr" sortable>Dst Address </td>
- <td colId="srcPort" sortable>Src Port </td>
- <td colId="dstPort" sortable>Dst Port </td>
- <td colId="protocol" sortable>Protocol </td>
- <td colId="metadata" sortable>Metadata </td>
+ <td colId="id" sortable>ID</td>
+ <td colId="srcAddr" sortable>Src Address</td>
+ <td colId="dstAddr" sortable>Dst Address</td>
+ <td colId="srcPort" sortable>Src Port</td>
+ <td colId="dstPort" sortable>Dst Port</td>
+ <td colId="protocol" sortable>Protocol</td>
+ <td colId="metadata" sortable>Metadata</td>
</tr>
</table>
</div>
diff --git a/apps/inbandtelemetry/impl/BUILD b/apps/inbandtelemetry/impl/BUILD
new file mode 100644
index 0000000..b3a1d46
--- /dev/null
+++ b/apps/inbandtelemetry/impl/BUILD
@@ -0,0 +1,12 @@
+COMPILE_DEPS = CORE_DEPS + KRYO + [
+ "//core/store/serializers:onos-core-serializers",
+ "//pipelines/basic:onos-pipelines-basic",
+ "//apps/inbandtelemetry/api:onos-apps-inbandtelemetry-api",
+]
+
+TEST_DEPS = TEST_ADAPTERS
+
+osgi_jar_with_tests(
+ test_deps = TEST_DEPS,
+ deps = COMPILE_DEPS,
+)
diff --git a/apps/inbandtelemetry/impl/src/main/java/org/onosproject/inbandtelemetry/impl/IntManager.java b/apps/inbandtelemetry/impl/src/main/java/org/onosproject/inbandtelemetry/impl/IntManager.java
deleted file mode 100644
index 5cf32da..0000000
--- a/apps/inbandtelemetry/impl/src/main/java/org/onosproject/inbandtelemetry/impl/IntManager.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Copyright 2015-present Open Networking Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.onosproject.inbandtelemetry.impl;
-
-import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Component;
-import org.apache.felix.scr.annotations.Deactivate;
-import org.apache.felix.scr.annotations.Reference;
-import org.apache.felix.scr.annotations.ReferenceCardinality;
-import org.apache.felix.scr.annotations.Service;
-import org.onlab.util.KryoNamespace;
-import org.onosproject.core.ApplicationId;
-import org.onosproject.core.CoreService;
-import org.onosproject.inbandtelemetry.api.IntConfig;
-import org.onosproject.inbandtelemetry.api.IntIntent;
-import org.onosproject.inbandtelemetry.api.IntIntentId;
-import org.onosproject.inbandtelemetry.api.IntObjective;
-import org.onosproject.inbandtelemetry.api.IntProgrammable;
-import org.onosproject.inbandtelemetry.api.IntService;
-import org.onosproject.net.Device;
-import org.onosproject.net.DeviceId;
-import org.onosproject.net.device.DeviceService;
-import org.onosproject.net.flow.FlowRuleService;
-import org.onosproject.net.host.HostEvent;
-import org.onosproject.net.host.HostListener;
-import org.onosproject.net.host.HostService;
-import org.onosproject.store.serializers.KryoNamespaces;
-import org.onosproject.store.service.AtomicIdGenerator;
-import org.onosproject.store.service.ConsistentMap;
-import org.onosproject.store.service.Serializer;
-import org.onosproject.store.service.StorageService;
-import org.slf4j.Logger;
-
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import static org.slf4j.LoggerFactory.getLogger;
-
-/**
- * Implementation of IntService, for controlling INT-capable pipelines.
- */
-@Component(immediate = true)
-@Service
-public class IntManager implements IntService {
- private final String appName = "org.onosproject.inbandtelemetry";
- private ApplicationId appId;
- private final Logger log = getLogger(getClass());
- private ConsistentMap<IntIntentId, IntIntent> intentConsistentMap;
- private ConsistentMap<DeviceId, IntDeviceRole> deviceRoleConsistentMap;
- private IntConfig cfg;
- private AtomicIdGenerator intentIds;
-
- private InternalHostListener hostListener = new InternalHostListener();
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- private CoreService coreService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- private DeviceService deviceService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- private StorageService storageService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- private HostService hostService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- private FlowRuleService flowRuleService;
-
- @Activate
- public void activate() {
- appId = coreService.registerApplication(appName);
-
- KryoNamespace.Builder serializer = KryoNamespace.newBuilder()
- .register(KryoNamespaces.API)
- .register(IntIntent.class)
- .register(IntIntentId.class)
- .register(IntDeviceRole.class)
- .register(IntIntent.IntHeaderType.class)
- .register(IntIntent.IntMetadataType.class)
- .register(IntIntent.IntReportType.class)
- .register(IntIntent.TelemetryMode.class);
-
- intentConsistentMap = storageService.<IntIntentId, IntIntent>consistentMapBuilder()
- .withSerializer(Serializer.using(serializer.build()))
- .withName("int-intents")
- .withApplicationId(appId)
- .withPurgeOnUninstall()
- .build();
-
- deviceRoleConsistentMap = storageService.<DeviceId, IntDeviceRole>consistentMapBuilder()
- .withSerializer(Serializer.using(serializer.build()))
- .withName("int-device-roles")
- .withApplicationId(appId)
- .withPurgeOnUninstall()
- .build();
-
- // Assign IntDeviceRole to each device
- deviceService.getAvailableDevices().forEach(device ->
- deviceRoleConsistentMap.put(device.id(),
- hostService.getConnectedHosts(device.id()).isEmpty() ?
- IntDeviceRole.TRANSIT :
- IntDeviceRole.SOURCE_SINK)
- );
- hostService.addListener(hostListener);
- intentIds = storageService.getAtomicIdGenerator("int-intent-id-generator");
- startInt();
- log.info("Started", appId.id());
- }
-
- @Deactivate
- public void deactivate() {
- hostService.removeListener(hostListener);
- log.info("Deactivated");
- }
-
- @Override
- public void startInt() {
- deviceService.getAvailableDevices().forEach(device -> {
- if (device.is(IntProgrammable.class)) {
- IntProgrammable intDevice = device.as(IntProgrammable.class);
- intDevice.init();
- }
- });
- }
-
- @Override
- public void startInt(Set<DeviceId> deviceIds) {
- deviceIds.forEach(deviceId -> {
- Device device = deviceService.getDevice(deviceId);
- if (device.is(IntProgrammable.class) &&
- getIntRole(deviceId) == IntDeviceRole.TRANSIT) {
- IntProgrammable intDevice = device.as(IntProgrammable.class);
- intDevice.init();
- }
- });
- }
-
- @Override
- public void stopInt() {
- flowRuleService.removeFlowRulesById(appId);
- }
-
- @Override
- public void stopInt(Set<DeviceId> deviceIds) {
-
- }
-
- @Override
- public void setConfig(IntConfig cfg) {
- this.cfg = cfg;
- deviceService.getAvailableDevices().forEach(device -> {
- if (device.is(IntProgrammable.class)) {
- IntProgrammable intDevice = device.as(IntProgrammable.class);
- intDevice.setupIntConfig(cfg);
- }
- });
- }
-
- @Override
- public IntConfig getConfig() {
- return cfg;
- }
-
- @Override
- public IntIntentId installIntIntent(IntIntent intent) {
- Integer intentId = (int) intentIds.nextId();
- IntIntentId intIntentId = IntIntentId.valueOf(intentId);
- intentConsistentMap.put(intIntentId, intent);
-
- // Convert IntIntent into an IntObjective
- IntObjective obj = new IntObjective.Builder()
- .withSelector(intent.selector())
- .withMetadataTypes(intent.metadataTypes())
- .withHeaderType(intent.headerType())
- .build();
-
- // Install IntObjective on each INT source device
- deviceService.getAvailableDevices().forEach(device -> {
- if (device.is(IntProgrammable.class)
- && deviceRoleConsistentMap.get(device.id()).value() == IntDeviceRole.SOURCE_SINK) {
- IntProgrammable intDevice = device.as(IntProgrammable.class);
- intDevice.addIntObjective(obj);
- }
- });
- return intIntentId;
- }
-
- @Override
- public void removeIntIntent(IntIntentId intentId) {
- IntIntent intent = intentConsistentMap.remove(intentId).value();
-
- // Convert IntIntent into an IntObjective
- IntObjective obj = new IntObjective.Builder()
- .withSelector(intent.selector())
- .withMetadataTypes(intent.metadataTypes())
- .withHeaderType(intent.headerType())
- .build();
-
- // Remove IntObjective on each INT source device
- deviceService.getAvailableDevices().forEach(device -> {
- if (device.is(IntProgrammable.class)
- && deviceRoleConsistentMap.get(device.id()).value() == IntDeviceRole.SOURCE_SINK) {
- IntProgrammable intDevice = device.as(IntProgrammable.class);
- intDevice.removeIntObjective(obj);
- }
- });
- }
-
- @Override
- public IntIntent getIntIntent(IntIntentId intentId) {
- return Optional.ofNullable(intentConsistentMap.get(intentId).value()).orElse(null);
- }
-
- @Override
- public Map<IntIntentId, IntIntent> getIntIntents() {
- return intentConsistentMap.asJavaMap();
- }
-
- private IntDeviceRole getIntRole(DeviceId deviceId) {
- return deviceRoleConsistentMap.get(deviceId).value();
- }
-
- private void setIntRole(DeviceId deviceId, IntDeviceRole role) {
- deviceRoleConsistentMap.put(deviceId, role);
- }
-
- private class InternalHostListener implements HostListener {
- @Override
- public void event(HostEvent event) {
- DeviceId deviceId = event.subject().location().deviceId();
- if (!deviceService.getDevice(deviceId).is(IntProgrammable.class)) {
- return;
- }
- switch (event.type()) {
- case HOST_ADDED:
- // When a host is attached to the switch, we can configure it
- // to work as SOURCE_SINK switch.
- if (deviceRoleConsistentMap.getOrDefault(deviceId, IntDeviceRole.TRANSIT).value()
- != IntDeviceRole.SOURCE_SINK) {
- setIntRole(deviceId, IntDeviceRole.SOURCE_SINK);
- }
- break;
- default:
- break;
- }
- }
- }
-
-}
diff --git a/apps/inbandtelemetry/impl/src/main/java/org/onosproject/inbandtelemetry/impl/SimpleIntManager.java b/apps/inbandtelemetry/impl/src/main/java/org/onosproject/inbandtelemetry/impl/SimpleIntManager.java
new file mode 100644
index 0000000..56ee52b
--- /dev/null
+++ b/apps/inbandtelemetry/impl/src/main/java/org/onosproject/inbandtelemetry/impl/SimpleIntManager.java
@@ -0,0 +1,527 @@
+/*
+ * Copyright 2015-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.inbandtelemetry.impl;
+
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.Striped;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
+import org.onlab.util.SharedScheduledExecutors;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.inbandtelemetry.api.IntConfig;
+import org.onosproject.inbandtelemetry.api.IntIntent;
+import org.onosproject.inbandtelemetry.api.IntIntentId;
+import org.onosproject.inbandtelemetry.api.IntObjective;
+import org.onosproject.inbandtelemetry.api.IntProgrammable;
+import org.onosproject.inbandtelemetry.api.IntService;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.MastershipRole;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceEvent;
+import org.onosproject.net.device.DeviceListener;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.host.HostEvent;
+import org.onosproject.net.host.HostListener;
+import org.onosproject.net.host.HostService;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.AtomicIdGenerator;
+import org.onosproject.store.service.AtomicValue;
+import org.onosproject.store.service.AtomicValueEvent;
+import org.onosproject.store.service.AtomicValueEventListener;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.MapEvent;
+import org.onosproject.store.service.MapEventListener;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.Versioned;
+import org.slf4j.Logger;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.Lock;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Simple implementation of IntService, for controlling INT-capable pipelines.
+ * <p>
+ * All INT intents are converted to an equivalent INT objective and applied to
+ * all SOURCE_SINK devices. A device is deemed SOURCE_SINK if it has at least
+ * one host attached.
+ * <p>
+ * The implementation listens for different types of events and when required it
+ * configures a device by cleaning-up any previous state and applying the new
+ * one.
+ */
+@Component(immediate = true)
+@Service
+public class SimpleIntManager implements IntService {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final int CONFIG_EVENT_DELAY = 5; // Seconds.
+
+ private static final String APP_NAME = "org.onosproject.inbandtelemetry";
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ private CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ private DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ private StorageService storageService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ private MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ private HostService hostService;
+
+ private final Striped<Lock> deviceLocks = Striped.lock(10);
+
+ private final ConcurrentMap<DeviceId, ScheduledFuture<?>> scheduledDeviceTasks = Maps.newConcurrentMap();
+
+ // Distributed state.
+ private ConsistentMap<IntIntentId, IntIntent> intentMap;
+ private ConsistentMap<DeviceId, Long> devicesToConfigure;
+ private AtomicValue<IntConfig> intConfig;
+ private AtomicValue<Boolean> intStarted;
+ private AtomicIdGenerator intentIds;
+
+ // Event listeners.
+ private final InternalHostListener hostListener = new InternalHostListener();
+ private final InternalDeviceListener deviceListener = new InternalDeviceListener();
+ private final InternalIntentMapListener intentMapListener = new InternalIntentMapListener();
+ private final InternalIntConfigListener intConfigListener = new InternalIntConfigListener();
+ private final InternalIntStartedListener intStartedListener = new InternalIntStartedListener();
+ private final InternalDeviceToConfigureListener devicesToConfigureListener =
+ new InternalDeviceToConfigureListener();
+
+ @Activate
+ public void activate() {
+
+ final ApplicationId appId = coreService.registerApplication(APP_NAME);
+
+ KryoNamespace.Builder serializer = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(IntIntent.class)
+ .register(IntIntentId.class)
+ .register(IntDeviceRole.class)
+ .register(IntIntent.IntHeaderType.class)
+ .register(IntIntent.IntMetadataType.class)
+ .register(IntIntent.IntReportType.class)
+ .register(IntIntent.TelemetryMode.class)
+ .register(IntConfig.class)
+ .register(IntConfig.TelemetrySpec.class);
+
+ devicesToConfigure = storageService.<DeviceId, Long>consistentMapBuilder()
+ .withSerializer(Serializer.using(serializer.build()))
+ .withName("onos-int-devices-to-configure")
+ .withApplicationId(appId)
+ .withPurgeOnUninstall()
+ .build();
+ devicesToConfigure.addListener(devicesToConfigureListener);
+
+ intentMap = storageService.<IntIntentId, IntIntent>consistentMapBuilder()
+ .withSerializer(Serializer.using(serializer.build()))
+ .withName("onos-int-intents")
+ .withApplicationId(appId)
+ .withPurgeOnUninstall()
+ .build();
+ intentMap.addListener(intentMapListener);
+
+ intStarted = storageService.<Boolean>atomicValueBuilder()
+ .withSerializer(Serializer.using(serializer.build()))
+ .withName("onos-int-started")
+ .withApplicationId(appId)
+ .build()
+ .asAtomicValue();
+ intStarted.addListener(intStartedListener);
+
+ intConfig = storageService.<IntConfig>atomicValueBuilder()
+ .withSerializer(Serializer.using(serializer.build()))
+ .withName("onos-int-config")
+ .withApplicationId(appId)
+ .build()
+ .asAtomicValue();
+ intConfig.addListener(intConfigListener);
+
+ intentIds = storageService.getAtomicIdGenerator("int-intent-id-generator");
+
+ // Bootstrap config for already existing devices.
+ triggerAllDeviceConfigure();
+
+ hostService.addListener(hostListener);
+ deviceService.addListener(deviceListener);
+
+ startInt();
+ log.info("Started", appId.id());
+ }
+
+ @Deactivate
+ public void deactivate() {
+ deviceService.removeListener(deviceListener);
+ hostService.removeListener(hostListener);
+ intentIds = null;
+ intConfig.removeListener(intConfigListener);
+ intConfig = null;
+ intStarted.removeListener(intStartedListener);
+ intStarted = null;
+ intentMap.removeListener(intentMapListener);
+ intentMap = null;
+ devicesToConfigure.removeListener(devicesToConfigureListener);
+ devicesToConfigure.destroy();
+ devicesToConfigure = null;
+ // Cancel tasks (if any).
+ scheduledDeviceTasks.values().forEach(f -> {
+ f.cancel(true);
+ if (!f.isDone()) {
+ try {
+ f.get(1, TimeUnit.SECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ // Don't care, we are terminating the service anyways.
+ }
+ }
+ });
+ // Clean up INT rules from existing devices.
+ deviceService.getDevices().forEach(d -> cleanupDevice(d.id()));
+ log.info("Deactivated");
+ }
+
+ @Override
+ public void startInt() {
+ // Atomic value event will trigger device configure.
+ intStarted.set(true);
+ }
+
+ @Override
+ public void startInt(Set<DeviceId> deviceIds) {
+ log.warn("Starting INT for a subset of devices is not supported");
+ }
+
+ @Override
+ public void stopInt() {
+ // Atomic value event will trigger device configure.
+ intStarted.set(false);
+ }
+
+ @Override
+ public void stopInt(Set<DeviceId> deviceIds) {
+ log.warn("Stopping INT for a subset of devices is not supported");
+ }
+
+ @Override
+ public void setConfig(IntConfig cfg) {
+ checkNotNull(cfg);
+ // Atomic value event will trigger device configure.
+ intConfig.set(cfg);
+ }
+
+ @Override
+ public IntConfig getConfig() {
+ return intConfig.get();
+ }
+
+ @Override
+ public IntIntentId installIntIntent(IntIntent intent) {
+ checkNotNull(intent);
+ final Integer intentId = (int) intentIds.nextId();
+ final IntIntentId intIntentId = IntIntentId.valueOf(intentId);
+ // Intent map event will trigger device configure.
+ intentMap.put(intIntentId, intent);
+ return intIntentId;
+ }
+
+ @Override
+ public void removeIntIntent(IntIntentId intentId) {
+ checkNotNull(intentId);
+ // Intent map event will trigger device configure.
+ intentMap.remove(intentId).value();
+ }
+
+ @Override
+ public IntIntent getIntIntent(IntIntentId intentId) {
+ return Optional.ofNullable(intentMap.get(intentId).value()).orElse(null);
+ }
+
+ @Override
+ public Map<IntIntentId, IntIntent> getIntIntents() {
+ return intentMap.asJavaMap();
+ }
+
+ private boolean isConfigTaskValid(DeviceId deviceId, long creationTime) {
+ Versioned<?> versioned = devicesToConfigure.get(deviceId);
+ return versioned != null && versioned.creationTime() == creationTime;
+ }
+
+ private boolean isIntStarted() {
+ return intStarted.get();
+ }
+
+ private boolean isNotIntConfigured() {
+ return intConfig.get() == null;
+ }
+
+ private boolean isIntProgrammable(DeviceId deviceId) {
+ final Device device = deviceService.getDevice(deviceId);
+ return device != null && device.is(IntProgrammable.class);
+ }
+
+ private void triggerDeviceConfigure(DeviceId deviceId) {
+ if (isIntProgrammable(deviceId)) {
+ devicesToConfigure.put(deviceId, System.nanoTime());
+ }
+ }
+
+ private void triggerAllDeviceConfigure() {
+ deviceService.getDevices().forEach(d -> triggerDeviceConfigure(d.id()));
+ }
+
+ private void configDeviceTask(DeviceId deviceId, long creationTime) {
+ if (isConfigTaskValid(deviceId, creationTime)) {
+ // Task outdated.
+ return;
+ }
+ if (!deviceService.isAvailable(deviceId)) {
+ return;
+ }
+ final MastershipRole role = mastershipService.requestRoleForSync(deviceId);
+ if (!role.equals(MastershipRole.MASTER)) {
+ return;
+ }
+ deviceLocks.get(deviceId).lock();
+ try {
+ // Clean up first.
+ cleanupDevice(deviceId);
+ if (!configDevice(deviceId)) {
+ // Clean up if fails.
+ cleanupDevice(deviceId);
+ return;
+ }
+ devicesToConfigure.remove(deviceId);
+ } finally {
+ deviceLocks.get(deviceId).unlock();
+ }
+ }
+
+ private void cleanupDevice(DeviceId deviceId) {
+ final Device device = deviceService.getDevice(deviceId);
+ if (device == null || !device.is(IntProgrammable.class)) {
+ return;
+ }
+ device.as(IntProgrammable.class).cleanup();
+ }
+
+ private boolean configDevice(DeviceId deviceId) {
+ // Returns true if config was successful, false if not and a clean up is
+ // needed.
+ final Device device = deviceService.getDevice(deviceId);
+ if (device == null || !device.is(IntProgrammable.class)) {
+ return true;
+ }
+
+ if (isNotIntConfigured()) {
+ log.warn("Missing INT config, aborting programming of INT device {}", deviceId);
+ return true;
+ }
+
+ final boolean isEdge = !hostService.getConnectedHosts(deviceId).isEmpty();
+ final IntDeviceRole intDeviceRole = isEdge
+ ? IntDeviceRole.SOURCE_SINK
+ : IntDeviceRole.TRANSIT;
+
+ log.info("Started programming of INT device {} with role {}...",
+ deviceId, intDeviceRole);
+
+ final IntProgrammable intProg = device.as(IntProgrammable.class);
+
+ if (!isIntStarted()) {
+ // Leave device with no INT configuration.
+ return true;
+ }
+
+ if (!intProg.init()) {
+ log.warn("Unable to init INT pipeline on {}", deviceId);
+ return false;
+ }
+
+ if (intDeviceRole != IntDeviceRole.SOURCE_SINK) {
+ // Stop here, no more configuration needed for transit devices.
+ return true;
+ }
+
+ if (intProg.supportsFunctionality(IntProgrammable.IntFunctionality.SINK)) {
+ if (!intProg.setupIntConfig(intConfig.get())) {
+ log.warn("Unable to apply INT report config on {}", deviceId);
+ return false;
+ }
+ }
+
+ // Port configuration.
+ final Set<PortNumber> hostPorts = deviceService.getPorts(deviceId)
+ .stream()
+ .map(port -> new ConnectPoint(deviceId, port.number()))
+ .filter(cp -> !hostService.getConnectedHosts(cp).isEmpty())
+ .map(ConnectPoint::port)
+ .collect(Collectors.toSet());
+
+ for (PortNumber port : hostPorts) {
+ if (intProg.supportsFunctionality(IntProgrammable.IntFunctionality.SOURCE)) {
+ log.info("Setting port {}/{} as INT source port...", deviceId, port);
+ if (!intProg.setSourcePort(port)) {
+ log.warn("Unable to set INT source port {} on {}", port, deviceId);
+ return false;
+ }
+ }
+ if (intProg.supportsFunctionality(IntProgrammable.IntFunctionality.SINK)) {
+ log.info("Setting port {}/{} as INT sink port...", deviceId, port);
+ if (!intProg.setSinkPort(port)) {
+ log.warn("Unable to set INT sink port {} on {}", port, deviceId);
+ return false;
+ }
+ }
+ }
+
+ if (!intProg.supportsFunctionality(IntProgrammable.IntFunctionality.SOURCE)) {
+ // Stop here, no more configuration needed for sink devices.
+ return true;
+ }
+
+ // Apply intents.
+ // This is a trivial implementation where we simply get the
+ // corresponding INT objective from an intent and we apply to all source
+ // device.
+ final Collection<IntObjective> objectives = intentMap.values().stream()
+ .map(v -> getIntObjective(v.value()))
+ .collect(Collectors.toList());
+ int appliedCount = 0;
+ for (IntObjective objective : objectives) {
+ if (intProg.addIntObjective(objective)) {
+ appliedCount = appliedCount + 1;
+ }
+ }
+
+ log.info("Completed programming of {}, applied {} INT objectives of {} total",
+ deviceId, appliedCount, objectives.size());
+
+ return true;
+ }
+
+ private IntObjective getIntObjective(IntIntent intent) {
+ return new IntObjective.Builder()
+ .withSelector(intent.selector())
+ .withMetadataTypes(intent.metadataTypes())
+ .withHeaderType(intent.headerType())
+ .build();
+ }
+
+ /* Event listeners which trigger device configuration. */
+
+ private class InternalHostListener implements HostListener {
+ @Override
+ public void event(HostEvent event) {
+ final DeviceId deviceId = event.subject().location().deviceId();
+ triggerDeviceConfigure(deviceId);
+ }
+ }
+
+ private class InternalDeviceListener implements DeviceListener {
+ @Override
+ public void event(DeviceEvent event) {
+ switch (event.type()) {
+ case DEVICE_ADDED:
+ case DEVICE_UPDATED:
+ case DEVICE_REMOVED:
+ case DEVICE_SUSPENDED:
+ case DEVICE_AVAILABILITY_CHANGED:
+ case PORT_ADDED:
+ case PORT_UPDATED:
+ case PORT_REMOVED:
+ triggerDeviceConfigure(event.subject().id());
+ return;
+ case PORT_STATS_UPDATED:
+ return;
+ default:
+ log.warn("Unknown device event type {}", event.type());
+ }
+ }
+ }
+
+ private class InternalIntentMapListener
+ implements MapEventListener<IntIntentId, IntIntent> {
+ @Override
+ public void event(MapEvent<IntIntentId, IntIntent> event) {
+ triggerAllDeviceConfigure();
+ }
+ }
+
+ private class InternalIntConfigListener
+ implements AtomicValueEventListener<IntConfig> {
+ @Override
+ public void event(AtomicValueEvent<IntConfig> event) {
+ triggerAllDeviceConfigure();
+ }
+ }
+
+ private class InternalIntStartedListener
+ implements AtomicValueEventListener<Boolean> {
+ @Override
+ public void event(AtomicValueEvent<Boolean> event) {
+ triggerAllDeviceConfigure();
+ }
+ }
+
+ private class InternalDeviceToConfigureListener
+ implements MapEventListener<DeviceId, Long> {
+ @Override
+ public void event(MapEvent<DeviceId, Long> event) {
+ if (event.type().equals(MapEvent.Type.REMOVE) ||
+ event.newValue() == null) {
+ return;
+ }
+ // Schedule task in the future. Wait for events for this device to
+ // stabilize.
+ final DeviceId deviceId = event.key();
+ final long creationTime = event.newValue().creationTime();
+ ScheduledFuture<?> newTask = SharedScheduledExecutors.newTimeout(
+ () -> configDeviceTask(deviceId, creationTime),
+ CONFIG_EVENT_DELAY, TimeUnit.SECONDS);
+ ScheduledFuture<?> oldTask = scheduledDeviceTasks.put(deviceId, newTask);
+ if (oldTask != null) {
+ oldTask.cancel(false);
+ }
+ }
+ }
+}