Inter-connect k8s and openstack intg bridge, kbr-ex and kbr-router
Change-Id: Id7d3c874e8b267252ca387b1ca6f67b9f9bc5116
(cherry picked from commit 019ce6a7143620fab32b7b0579aa7381aa102af2)
diff --git a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
index db6583a..ecec966 100644
--- a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
+++ b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
@@ -132,6 +132,9 @@
public static final int EXT_ENTRY_TABLE = 0;
public static final int POD_RESOLUTION_TABLE = 11;
+ public static final int ROUTER_EXTRY_TABLE = 0;
+ public static final int EXT_RESOLUTION_TABLE = 11;
+
public static final int LOCAL_ENTRY_TABLE = 0;
public static final int TUN_ENTRY_TABLE = 0;
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
index 5fd2758..56fe815 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
@@ -33,10 +33,15 @@
import org.onosproject.k8snetworking.api.K8sNetworkService;
import org.onosproject.k8snetworking.api.K8sPort;
import org.onosproject.k8snetworking.util.RulePopulatorUtil;
+import org.onosproject.k8snode.api.K8sHost;
+import org.onosproject.k8snode.api.K8sHostEvent;
+import org.onosproject.k8snode.api.K8sHostListener;
+import org.onosproject.k8snode.api.K8sHostService;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
+import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.mastership.MastershipService;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
@@ -64,14 +69,17 @@
import static org.onosproject.k8snetworking.api.Constants.EXT_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.POD_RESOLUTION_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_DEFAULT_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
+import static org.onosproject.k8snetworking.api.Constants.ROUTER_EXTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
import static org.onosproject.k8snode.api.Constants.DEFAULT_EXTERNAL_GATEWAY_MAC;
+import static org.onosproject.k8snode.api.K8sApiConfig.Mode.PASSTHROUGH;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -115,12 +123,17 @@
protected K8sNodeService k8sNodeService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sHostService k8sHostService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
protected K8sFlowRuleService k8sFlowRuleService;
private final InternalK8sNetworkListener k8sNetworkListener =
new InternalK8sNetworkListener();
private final InternalK8sNodeListener k8sNodeListener =
new InternalK8sNodeListener();
+ private final InternalK8sHostListener k8sHostListener =
+ new InternalK8sHostListener();
private final ExecutorService eventExecutor = newSingleThreadExecutor(
groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
@@ -135,12 +148,14 @@
leadershipService.runForLeadership(appId.name());
k8sNetworkService.addListener(k8sNetworkListener);
k8sNodeService.addListener(k8sNodeListener);
+ k8sHostService.addListener(k8sHostListener);
log.info("Started");
}
@Deactivate
protected void deactivate() {
+ k8sHostService.removeListener(k8sHostListener);
k8sNodeService.removeListener(k8sNodeListener);
k8sNetworkService.removeListener(k8sNetworkListener);
leadershipService.withdraw(appId.name());
@@ -201,8 +216,8 @@
install);
}
- private void setSnatDownstreamRule(K8sNode k8sNode,
- boolean install) {
+ private void setExtSnatDownstreamRule(K8sNode k8sNode,
+ boolean install) {
DeviceId deviceId = k8sNode.extBridge();
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
@@ -231,8 +246,8 @@
install);
}
- private void setSnatUpstreamRule(K8sNode k8sNode,
- boolean install) {
+ private void setExtSnatUpstreamRule(K8sNode k8sNode,
+ boolean install) {
K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());
@@ -263,11 +278,15 @@
.setEthSrc(k8sNode.extBridgeMac())
.setEthDst(k8sNode.extGatewayMac());
- if (MacAddress.valueOf(DEFAULT_EXTERNAL_GATEWAY_MAC).equals(
- k8sNode.extGatewayMac())) {
- tBuilder.setOutput(k8sNode.extIntfPortNum());
+ if (k8sNode.mode() == PASSTHROUGH) {
+ tBuilder.setOutput(k8sNode.extToRouterPortNum());
} else {
- tBuilder.setOutput(k8sNode.extBridgePortNum());
+ if (MacAddress.valueOf(DEFAULT_EXTERNAL_GATEWAY_MAC).equals(
+ k8sNode.extGatewayMac())) {
+ tBuilder.setOutput(k8sNode.extIntfPortNum());
+ } else {
+ tBuilder.setOutput(k8sNode.extBridgePortNum());
+ }
}
}
@@ -312,6 +331,52 @@
});
}
+ private void setRouterSnatUpstreamRule(K8sNode k8sNode,
+ K8sRouterBridge bridge,
+ boolean install) {
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchInPort(k8sNode.routerToExtPortNum())
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(k8sNode.routerPortNum())
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ bridge.deviceId(),
+ selector,
+ treatment,
+ PRIORITY_DEFAULT_RULE,
+ ROUTER_EXTRY_TABLE,
+ install);
+ }
+
+ private void setRouterSnatDownstreamRule(K8sNode k8sNode,
+ K8sRouterBridge bridge,
+ boolean install) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchInPort(k8sNode.routerPortNum())
+ .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), 32))
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(k8sNode.routerToExtPortNum())
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ bridge.deviceId(),
+ selector,
+ treatment,
+ PRIORITY_DEFAULT_RULE,
+ ROUTER_EXTRY_TABLE,
+ install);
+ }
+
private class InternalK8sNodeListener implements K8sNodeListener {
private boolean isRelevantHelper() {
@@ -339,13 +404,54 @@
}
setExtIntfArpRule(k8sNode, true);
- setSnatDownstreamRule(k8sNode, true);
+ setExtSnatDownstreamRule(k8sNode, true);
setContainerToExtRule(k8sNode, true);
}
private void processNodeUpdate(K8sNode k8sNode) {
if (k8sNode.extGatewayMac() != null) {
- setSnatUpstreamRule(k8sNode, true);
+ setExtSnatUpstreamRule(k8sNode, true);
+ }
+ }
+ }
+
+ private class InternalK8sHostListener implements K8sHostListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sHostEvent event) {
+ switch (event.type()) {
+ case K8S_HOST_COMPLETE:
+ eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+ break;
+ case K8S_HOST_INCOMPLETE:
+ default:
+ break;
+ }
+ }
+
+ private void processNodeCompletion(K8sHost k8sHost) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ for (String name : k8sHost.nodeNames()) {
+ K8sNode node = k8sNodeService.node(name);
+ if (node == null) {
+ return;
+ }
+ K8sRouterBridge bridge = k8sHost.routerBridges().stream()
+ .filter(b -> b.segmentId() == node.segmentId())
+ .findAny().orElse(null);
+ if (bridge == null) {
+ return;
+ }
+
+ setRouterSnatUpstreamRule(node, bridge, true);
+ setRouterSnatDownstreamRule(node, bridge, true);
}
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 03a953e..dde6828 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -158,7 +158,7 @@
if (node.hostname().equals(k8sNetwork.name())) {
tBuilder.setEthDst(node.intgBridgeMac())
- .setOutput(PortNumber.LOCAL);
+ .setOutput(node.intgEntryPortNum());
} else {
K8sNode localNode = k8sNodeService.node(k8sNetwork.name());
@@ -198,7 +198,7 @@
if (node.hostname().equals(k8sNetwork.name())) {
sBuilder = DefaultTrafficSelector.builder()
- .matchInPort(PortNumber.LOCAL)
+ .matchInPort(node.intgEntryPortNum())
.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
HOST_PREFIX));
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
index d62a923..a0ccf5d 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
@@ -349,7 +349,7 @@
private void setLocalTunnelTagFlowRules(K8sNode k8sNode, boolean install) {
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchInPort(PortNumber.LOCAL)
+ .matchInPort(k8sNode.intgEntryPortNum())
.build();
K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
index 82b63cb..59a5715 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
@@ -23,19 +23,23 @@
private Constants() {
}
+ public static final String OS_INTEGRATION_BRIDGE = "br-int";
public static final String INTEGRATION_BRIDGE = "kbr-int";
public static final String EXTERNAL_BRIDGE = "kbr-ex";
public static final String LOCAL_BRIDGE = "kbr-local";
public static final String TUNNEL_BRIDGE = "kbr-tun";
- public static final String EXTERNAL_ROUTER = "kbr-router";
+ public static final String ROUTER_BRIDGE = "kbr-router";
public static final String INTEGRATION_TO_EXTERNAL_BRIDGE = "int-ex";
public static final String PHYSICAL_EXTERNAL_BRIDGE = "ex-int";
public static final String INTEGRATION_TO_LOCAL_BRIDGE = "int-local";
public static final String LOCAL_TO_INTEGRATION_BRIDGE = "local-int";
public static final String EXTERNAL_TO_ROUTER = "ex-router";
public static final String ROUTER_TO_EXTERNAL = "router-ex";
+ public static final String ROUTER = "router";
public static final String INTEGRATION_TO_TUN_BRIDGE = "int-tun";
public static final String TUN_TO_INTEGRATION_BRIDGE = "tun-int";
+ public static final String OS_TO_K8S_BRIDGE = "os-k8s";
+ public static final String K8S_TO_OS_BRIDGE = "k8s-os";
public static final String VXLAN_TUNNEL = "vxlan";
public static final String GRE_TUNNEL = "gre";
public static final String GENEVE_TUNNEL = "geneve";
@@ -50,4 +54,5 @@
public static final int DEFAULT_SEGMENT_ID = 100;
public static final String DEFAULT_EXTERNAL_GATEWAY_MAC = "fa:00:00:00:00:01";
public static final String DEFAULT_EXTERNAL_BRIDGE_MAC = "fa:00:00:00:00:02";
+ public static final String DEFAULT_INTG_BRIDGE_MAC = "fa:00:00:00:00:03";
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sHost.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sHost.java
index e961910..a6e4d0e 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sHost.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sHost.java
@@ -35,6 +35,7 @@
private final Set<String> nodeNames;
private final K8sHostState state;
private final Set<K8sTunnelBridge> tunBridges;
+ private final Set<K8sRouterBridge> routerBridges;
private static final String NOT_NULL_MSG = "Host % cannot be null";
@@ -47,13 +48,16 @@
* @param nodeNames node names
* @param state host state
* @param tunBridges a set of tunnel bridges
+ * @param routerBridges a set of router bridges
*/
protected DefaultK8sHost(IpAddress hostIp, Set<String> nodeNames,
- K8sHostState state, Set<K8sTunnelBridge> tunBridges) {
+ K8sHostState state, Set<K8sTunnelBridge> tunBridges,
+ Set<K8sRouterBridge> routerBridges) {
this.hostIp = hostIp;
this.nodeNames = nodeNames;
this.state = state;
this.tunBridges = tunBridges;
+ this.routerBridges = routerBridges;
}
@Override
@@ -82,12 +86,18 @@
}
@Override
+ public Set<K8sRouterBridge> routerBridges() {
+ return ImmutableSet.copyOf(routerBridges);
+ }
+
+ @Override
public K8sHost updateState(K8sHostState newState) {
return new Builder()
.hostIp(hostIp)
.nodeNames(nodeNames)
.state(newState)
.tunBridges(tunBridges)
+ .routerBridges(routerBridges)
.build();
}
@@ -98,6 +108,7 @@
.nodeNames(nodeNames)
.state(state)
.tunBridges(tunBridges)
+ .routerBridges(routerBridges)
.build();
}
@@ -113,12 +124,13 @@
return Objects.equals(hostIp, that.hostIp) &&
Objects.equals(nodeNames, that.nodeNames) &&
state == that.state &&
- Objects.equals(tunBridges, that.tunBridges);
+ Objects.equals(tunBridges, that.tunBridges) &&
+ Objects.equals(routerBridges, that.routerBridges);
}
@Override
public int hashCode() {
- return Objects.hash(hostIp, nodeNames, state);
+ return Objects.hash(hostIp, nodeNames, state, tunBridges, routerBridges);
}
@Override
@@ -128,6 +140,7 @@
.add("nodeNames", nodeNames)
.add("state", state)
.add("tunBridges", tunBridges)
+ .add("routerBridges", routerBridges)
.toString();
}
@@ -146,6 +159,7 @@
private Set<String> nodeNames;
private K8sHostState state;
private Set<K8sTunnelBridge> tunBridges;
+ private Set<K8sRouterBridge> routerBridges;
// private constructor not intended to use from external
private Builder() {
@@ -164,7 +178,11 @@
tunBridges = new HashSet<>();
}
- return new DefaultK8sHost(hostIp, nodeNames, state, tunBridges);
+ if (routerBridges == null) {
+ routerBridges = new HashSet<>();
+ }
+
+ return new DefaultK8sHost(hostIp, nodeNames, state, tunBridges, routerBridges);
}
@Override
@@ -190,5 +208,11 @@
this.tunBridges = tunBridges;
return this;
}
+
+ @Override
+ public Builder routerBridges(Set<K8sRouterBridge> routerBridges) {
+ this.routerBridges = routerBridges;
+ return this;
+ }
}
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
index 6bb2001..09f4097 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
@@ -34,16 +34,22 @@
import static org.onosproject.k8snode.api.Constants.DEFAULT_CLUSTER_NAME;
import static org.onosproject.k8snode.api.Constants.DEFAULT_EXTERNAL_BRIDGE_MAC;
import static org.onosproject.k8snode.api.Constants.DEFAULT_EXTERNAL_GATEWAY_MAC;
+import static org.onosproject.k8snode.api.Constants.DEFAULT_INTG_BRIDGE_MAC;
import static org.onosproject.k8snode.api.Constants.EXTERNAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.EXTERNAL_TO_ROUTER;
import static org.onosproject.k8snode.api.Constants.GENEVE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.GRE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_LOCAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_TUN_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.K8S_TO_OS_BRIDGE;
import static org.onosproject.k8snode.api.Constants.LOCAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.LOCAL_TO_INTEGRATION_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.OS_TO_K8S_BRIDGE;
import static org.onosproject.k8snode.api.Constants.PHYSICAL_EXTERNAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.ROUTER;
+import static org.onosproject.k8snode.api.Constants.ROUTER_TO_EXTERNAL;
import static org.onosproject.k8snode.api.Constants.TUNNEL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.TUN_TO_INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.VXLAN_TUNNEL;
@@ -445,6 +451,63 @@
}
@Override
+ public PortNumber routerToExtPortNum() {
+ if (mode() == PASSTHROUGH) {
+ K8sHostService hostService =
+ DefaultServiceDirectory.getService(K8sHostService.class);
+ Port port = null;
+ for (K8sHost host : hostService.hosts()) {
+ if (host.nodeNames().contains(hostname())) {
+ for (K8sRouterBridge bridge : host.routerBridges()) {
+ if (bridge.segmentId() == segmentId()) {
+ port = port(bridge.deviceId(), routerToExtPatchPortName());
+ }
+ }
+ }
+ }
+
+ if (port == null) {
+ return null;
+ } else {
+ return port.number();
+ }
+ }
+
+ return null;
+ }
+
+ @Override
+ public PortNumber extToRouterPortNum() {
+ return portNumber(extBridge, extToRouterPatchPortName());
+ }
+
+ @Override
+ public PortNumber routerPortNum() {
+ if (mode() == PASSTHROUGH) {
+ K8sHostService hostService =
+ DefaultServiceDirectory.getService(K8sHostService.class);
+ Port port = null;
+ for (K8sHost host : hostService.hosts()) {
+ if (host.nodeNames().contains(hostname())) {
+ for (K8sRouterBridge bridge : host.routerBridges()) {
+ if (bridge.segmentId() == segmentId()) {
+ port = port(bridge.deviceId(), routerPortName());
+ }
+ }
+ }
+ }
+
+ if (port == null) {
+ return null;
+ } else {
+ return port.number();
+ }
+ }
+
+ return null;
+ }
+
+ @Override
public PortNumber extBridgePortNum() {
return PortNumber.LOCAL;
}
@@ -459,7 +522,11 @@
@Override
public MacAddress intgBridgeMac() {
- return macAddress(intgBridge, intgBridgeName());
+ if (mode == PASSTHROUGH) {
+ return MacAddress.valueOf(DEFAULT_INTG_BRIDGE_MAC);
+ } else {
+ return macAddress(intgBridge, intgBridgeName());
+ }
}
@Override
@@ -523,6 +590,24 @@
}
@Override
+ public String intgEntryPortName() {
+ if (mode == PASSTHROUGH) {
+ return k8sToOsIntgPatchPortName();
+ } else {
+ return intgBridgeName();
+ }
+ }
+
+ @Override
+ public PortNumber intgEntryPortNum() {
+ if (mode == PASSTHROUGH) {
+ return portNumber(intgBridge, k8sToOsIntgPatchPortName());
+ } else {
+ return intgBridgePortNum();
+ }
+ }
+
+ @Override
public String extBridgeName() {
if (mode == PASSTHROUGH) {
return EXTERNAL_BRIDGE + "-" + uniqueString(5);
@@ -640,6 +725,51 @@
}
@Override
+ public String k8sToOsIntgPatchPortName() {
+ if (mode == PASSTHROUGH) {
+ return K8S_TO_OS_BRIDGE + "-" + uniqueString(5);
+ } else {
+ return K8S_TO_OS_BRIDGE;
+ }
+ }
+
+ @Override
+ public String osToK8sIntgPatchPortName() {
+ if (mode == PASSTHROUGH) {
+ return OS_TO_K8S_BRIDGE + "-" + uniqueString(5);
+ } else {
+ return OS_TO_K8S_BRIDGE;
+ }
+ }
+
+ @Override
+ public String routerToExtPatchPortName() {
+ if (mode == PASSTHROUGH) {
+ return ROUTER_TO_EXTERNAL + "-" + uniqueString(5);
+ } else {
+ return ROUTER_TO_EXTERNAL;
+ }
+ }
+
+ @Override
+ public String extToRouterPatchPortName() {
+ if (mode == PASSTHROUGH) {
+ return EXTERNAL_TO_ROUTER + "-" + uniqueString(5);
+ } else {
+ return EXTERNAL_TO_ROUTER;
+ }
+ }
+
+ @Override
+ public String routerPortName() {
+ if (mode == PASSTHROUGH) {
+ return ROUTER + "-" + segmentId();
+ } else {
+ return ROUTER;
+ }
+ }
+
+ @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sBridge.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sBridge.java
new file mode 100644
index 0000000..7659689
--- /dev/null
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sBridge.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snode.api;
+
+import org.onosproject.net.DeviceId;
+
+/**
+ * K8s bridge interface.
+ */
+public interface K8sBridge {
+
+ /**
+ * Returns device identifier.
+ *
+ * @return device identifier
+ */
+ DeviceId deviceId();
+
+ /**
+ * Return the datapath identifier.
+ *
+ * @return datapath identifier
+ */
+ String dpid();
+
+ /**
+ * Returns bridge name.
+ *
+ * @return bridge name
+ */
+ String name();
+}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHost.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHost.java
index 2eeb9c1..ab33cfe 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHost.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHost.java
@@ -62,6 +62,13 @@
Set<K8sTunnelBridge> tunBridges();
/**
+ * Returns the set of router bridges belong to the host.
+ *
+ * @return a set of router bridges
+ */
+ Set<K8sRouterBridge> routerBridges();
+
+ /**
* Returns new kubernetes host instance with given state.
*
* @param newState updated state
@@ -120,5 +127,13 @@
* @return kubernetes host builder
*/
Builder tunBridges(Set<K8sTunnelBridge> tunBridges);
+
+ /**
+ * Returns kubernetes host builder with supplied router bridges set.
+ *
+ * @param routerBridges router bridges
+ * @return kubernetes host builder
+ */
+ Builder routerBridges(Set<K8sRouterBridge> routerBridges);
}
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
index 632f7d2..9b695eb 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
@@ -245,6 +245,20 @@
String intgBridgeName();
/**
+ * Returns the entry port name of integration bridge.
+ *
+ * @return entry port name
+ */
+ String intgEntryPortName();
+
+ /**
+ * Return the port number of integration bridge's entry port.
+ *
+ * @return port number
+ */
+ PortNumber intgEntryPortNum();
+
+ /**
* Returns external bridge name.
*
* @return external bridge name
@@ -336,6 +350,41 @@
String tunToIntgPatchPortName();
/**
+ * Returns kubernetes to openstack integration patch port name.
+ *
+ * @return kubernetes to openstack integration patch port name
+ */
+ String k8sToOsIntgPatchPortName();
+
+ /**
+ * Returns openstack to kubernetes integration patch port name.
+ *
+ * @return openstack to kubernetes integration patch port name
+ */
+ String osToK8sIntgPatchPortName();
+
+ /**
+ * Returns router to external bridge patch port name.
+ *
+ * @return router to external bridge patch port name
+ */
+ String routerToExtPatchPortName();
+
+ /**
+ * Returns external to router bridge patch port name.
+ *
+ * @return external to router bridge patch port name
+ */
+ String extToRouterPatchPortName();
+
+ /**
+ * Returns router part name.
+ *
+ * @return router port name
+ */
+ String routerPortName();
+
+ /**
* Returns the GRE tunnel port number.
*
* @return GRE port number; null if the GRE tunnel port does not exist
@@ -406,6 +455,27 @@
PortNumber tunToIntgPortNum();
/**
+ * Returns the router to external bridge patch port number.
+ *
+ * @return patch port number
+ */
+ PortNumber routerToExtPortNum();
+
+ /**
+ * Returns the external to router bridge patch port number.
+ *
+ * @return patch port number
+ */
+ PortNumber extToRouterPortNum();
+
+ /**
+ * Returns the router port number.
+ *
+ * @return router port number
+ */
+ PortNumber routerPortNum();
+
+ /**
* Returns the external bridge to router port number.
*
* @return port number, null if the port does not exist
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sRouterBridge.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sRouterBridge.java
new file mode 100644
index 0000000..7e5b472
--- /dev/null
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sRouterBridge.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2020-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snode.api;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Strings;
+import org.onosproject.net.DeviceId;
+
+import java.util.Objects;
+
+import static org.onosproject.k8snode.api.Constants.ROUTER_BRIDGE;
+
+/**
+ * K8s router bridge.
+ */
+public class K8sRouterBridge implements K8sBridge {
+
+ private static final String OF_PREFIX = "of:";
+
+ private final int segmentId;
+
+ /**
+ * Default constructor.
+ *
+ * @param segmentId segment identifier
+ */
+ public K8sRouterBridge(int segmentId) {
+ this.segmentId = segmentId;
+ }
+
+ /**
+ * Returns segment ID.
+ *
+ * @return segment ID
+ */
+ public int segmentId() {
+ return segmentId;
+ }
+
+ @Override
+ public DeviceId deviceId() {
+ return DeviceId.deviceId(dpid());
+ }
+
+ @Override
+ public String dpid() {
+ return genDpidFromName(name());
+ }
+
+ @Override
+ public String name() {
+ return ROUTER_BRIDGE + "-" + segmentId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ K8sRouterBridge that = (K8sRouterBridge) o;
+ return segmentId == that.segmentId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(segmentId);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("segmentId", segmentId)
+ .toString();
+ }
+
+ private String genDpidFromName(String name) {
+ if (name != null) {
+ String hexString = Integer.toHexString(name.hashCode());
+ return OF_PREFIX + Strings.padStart(hexString, 16, '0');
+ }
+
+ return null;
+ }
+}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sTunnelBridge.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sTunnelBridge.java
index 50d155e..510c9b7 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sTunnelBridge.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sTunnelBridge.java
@@ -29,7 +29,7 @@
/**
* K8s tunnel bridge.
*/
-public class K8sTunnelBridge {
+public class K8sTunnelBridge implements K8sBridge {
private static final String OF_PREFIX = "of:";
@@ -44,11 +44,7 @@
this.tunnelId = tunnelId;
}
- /**
- * Returns device identifier.
- *
- * @return device identifier
- */
+ @Override
public DeviceId deviceId() {
return DeviceId.deviceId(dpid());
}
@@ -62,20 +58,12 @@
return tunnelId;
}
- /**
- * Return the datapath identifier.
- *
- * @return datapath identifier
- */
+ @Override
public String dpid() {
return genDpidFromName(name());
}
- /**
- * Returns bridge name.
- *
- * @return bridge name
- */
+ @Override
public String name() {
return TUNNEL_BRIDGE + "-" + tunnelId;
}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sHostListCommand.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sHostListCommand.java
index fa996ba..010fcc1 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sHostListCommand.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sHostListCommand.java
@@ -24,6 +24,7 @@
import org.onosproject.cli.AbstractShellCommand;
import org.onosproject.k8snode.api.K8sHost;
import org.onosproject.k8snode.api.K8sHostService;
+import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.k8snode.api.K8sTunnelBridge;
import java.util.Comparator;
@@ -42,8 +43,8 @@
public class K8sHostListCommand extends AbstractShellCommand {
private static final int HOST_IP_LENGTH = 15;
- private static final int NODES_LENGTH = 40;
private static final int TUNBRS_LENGTH = 40;
+ private static final int RTRBRS_LENGTH = 40;
private static final int STATUS_LENGTH = 15;
@Override
@@ -53,18 +54,19 @@
hosts.sort(Comparator.comparing(K8sHost::hostIp));
String format = genFormatString(
- ImmutableList.of(HOST_IP_LENGTH, NODES_LENGTH, TUNBRS_LENGTH, STATUS_LENGTH));
+ ImmutableList.of(HOST_IP_LENGTH, TUNBRS_LENGTH, RTRBRS_LENGTH, STATUS_LENGTH));
if (outputJson()) {
print("%s", json(hosts));
} else {
- print(format, "Host IP", "Nodes", "Tunnel Bridges", "State");
+ print(format, "Host IP", "Tunnel Bridges", "Router Bridges", "State");
for (K8sHost host : hosts) {
print(format,
host.hostIp().toString(),
- host.nodeNames().toString(),
host.tunBridges().stream().map(K8sTunnelBridge::name)
.collect(Collectors.toSet()).toString(),
+ host.routerBridges().stream().map(K8sRouterBridge::name)
+ .collect(Collectors.toSet()).toString(),
host.state().toString());
}
print("Total %s hosts", hosts.size());
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
index cf78b6e..bf81f85a 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
@@ -39,6 +39,7 @@
import org.onosproject.k8snode.api.K8sHostState;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeAdminService;
+import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.k8snode.api.K8sTunnelBridge;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
@@ -186,12 +187,14 @@
private K8sHost buildK8sHost(HostNodesInfo hostNodesInfo, K8sApiConfig config) {
int segmentId = config.segmentId();
- K8sTunnelBridge bridge = new K8sTunnelBridge(segmentId);
+ K8sTunnelBridge tBridge = new K8sTunnelBridge(segmentId);
+ K8sRouterBridge rBridge = new K8sRouterBridge(segmentId);
return DefaultK8sHost.builder()
.hostIp(hostNodesInfo.hostIp())
.state(K8sHostState.INIT)
- .tunBridges(ImmutableSet.of(bridge))
+ .tunBridges(ImmutableSet.of(tBridge))
+ .routerBridges(ImmutableSet.of(rBridge))
.nodeNames(hostNodesInfo.nodes())
.build();
}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
index 5da9624..3827e21 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
@@ -20,6 +20,7 @@
import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
+import org.onosproject.k8snode.api.K8sBridge;
import org.onosproject.k8snode.api.K8sHost;
import org.onosproject.k8snode.api.K8sHostAdminService;
import org.onosproject.k8snode.api.K8sHostEvent;
@@ -28,6 +29,7 @@
import org.onosproject.k8snode.api.K8sHostState;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeAdminService;
+import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.k8snode.api.K8sTunnelBridge;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
@@ -68,6 +70,7 @@
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.k8snode.api.Constants.GENEVE;
import static org.onosproject.k8snode.api.Constants.GRE;
+import static org.onosproject.k8snode.api.Constants.OS_INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.VXLAN;
import static org.onosproject.k8snode.api.K8sHostState.COMPLETE;
import static org.onosproject.k8snode.api.K8sHostState.DEVICE_CREATED;
@@ -163,6 +166,12 @@
createBridge(k8sHost.ovsdb(), tunBridge);
}
}
+
+ for (K8sRouterBridge routerBridge : k8sHost.routerBridges()) {
+ if (!deviceService.isAvailable(routerBridge.deviceId())) {
+ createBridge(k8sHost.ovsdb(), routerBridge);
+ }
+ }
}
@Override
@@ -178,7 +187,8 @@
for (String node : k8sHost.nodeNames()) {
K8sNode k8sNode = k8sNodeAdminService.node(node);
if (k8sNode.segmentId() == bridge.tunnelId()) {
- createPatchInterfaces(k8sHost.ovsdb(), bridge, k8sNode);
+ createTunnelPatchInterfaces(k8sHost.ovsdb(), bridge, k8sNode);
+ createInterPatchInterfaces(k8sHost.ovsdb(), k8sNode);
}
}
}
@@ -197,6 +207,16 @@
createGeneveTunnelInterface(k8sHost.ovsdb(), bridge);
}
}
+
+ // create patch ports into router bridge face to external bridge
+ for (K8sRouterBridge bridge : k8sHost.routerBridges()) {
+ for (String node : k8sHost.nodeNames()) {
+ K8sNode k8sNode = k8sNodeAdminService.node(node);
+ if (k8sNode.segmentId() == bridge.segmentId()) {
+ createRouterPatchInterfaces(k8sHost.ovsdb(), bridge, k8sNode);
+ }
+ }
+ }
} catch (Exception e) {
log.error("Exception occurred because of {}", e);
}
@@ -212,7 +232,7 @@
// do something if needed
}
- private void createBridge(DeviceId ovsdb, K8sTunnelBridge bridge) {
+ private void createBridge(DeviceId ovsdb, K8sBridge bridge) {
Device device = deviceService.getDevice(ovsdb);
List<ControllerInfo> controllers = clusterService.getNodes().stream()
@@ -232,7 +252,7 @@
bridgeConfig.addBridge(builder.build());
}
- private void createPatchInterfaces(DeviceId ovsdb, K8sTunnelBridge bridge, K8sNode k8sNode) {
+ private void createTunnelPatchInterfaces(DeviceId ovsdb, K8sBridge bridge, K8sNode k8sNode) {
Device device = deviceService.getDevice(ovsdb);
if (device == null || !device.is(InterfaceConfig.class)) {
log.error("Failed to create patch interface on {}", ovsdb);
@@ -241,7 +261,7 @@
InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
- // tunnel bridge -> integration bridge
+ // tunnel bridge -> k8s integration bridge
PatchDescription brTunIntPatchDesc =
DefaultPatchDescription.builder()
.deviceId(bridge.name())
@@ -252,6 +272,46 @@
ifaceConfig.addPatchMode(k8sNode.tunToIntgPatchPortName(), brTunIntPatchDesc);
}
+ private void createInterPatchInterfaces(DeviceId ovsdb, K8sNode k8sNode) {
+ Device device = deviceService.getDevice(ovsdb);
+ if (device == null || !device.is(InterfaceConfig.class)) {
+ log.error("Failed to create patch interface on {}", ovsdb);
+ return;
+ }
+
+ InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
+
+ // openstack integration bridge -> k8s integration bridge
+ PatchDescription osIntK8sIntPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(OS_INTEGRATION_BRIDGE)
+ .ifaceName(k8sNode.osToK8sIntgPatchPortName())
+ .peer(k8sNode.k8sToOsIntgPatchPortName())
+ .build();
+
+ ifaceConfig.addPatchMode(k8sNode.tunToIntgPatchPortName(), osIntK8sIntPatchDesc);
+ }
+
+ private void createRouterPatchInterfaces(DeviceId ovsdb, K8sBridge bridge, K8sNode k8sNode) {
+ Device device = deviceService.getDevice(ovsdb);
+ if (device == null || !device.is(InterfaceConfig.class)) {
+ log.error("Failed to create patch interface on {}", ovsdb);
+ return;
+ }
+
+ InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
+
+ // router bridge -> external bridge
+ PatchDescription brRouterExtPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(bridge.name())
+ .ifaceName(k8sNode.routerToExtPatchPortName())
+ .peer(k8sNode.extToRouterPatchPortName())
+ .build();
+
+ ifaceConfig.addPatchMode(k8sNode.tunToIntgPatchPortName(), brRouterExtPatchDesc);
+ }
+
private void createVxlanTunnelInterface(DeviceId ovsdb, K8sTunnelBridge bridge) {
createTunnelInterface(ovsdb, bridge, VXLAN, bridge.vxlanPortName());
}
@@ -356,12 +416,18 @@
log.error("Exception caused during init state checking...");
}
- for (K8sTunnelBridge tunBridge : k8sHost.tunBridges()) {
+ for (K8sBridge tunBridge : k8sHost.tunBridges()) {
if (!deviceService.isAvailable(tunBridge.deviceId())) {
return false;
}
}
+ for (K8sBridge routerBridge: k8sHost.routerBridges()) {
+ if (!deviceService.isAvailable(routerBridge.deviceId())) {
+ return false;
+ }
+ }
+
return true;
}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
index cec3d2c..62db27d 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
@@ -408,6 +408,24 @@
.build();
ifaceConfig.addPatchMode(k8sNode.tunToIntgPatchPortName(), brTunIntPatchDesc);
+ } else {
+ // k8s integration bridge -> openstack integration bridge
+ PatchDescription k8sIntOsIntPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(k8sNode.intgBridgeName())
+ .ifaceName(k8sNode.k8sToOsIntgPatchPortName())
+ .peer(k8sNode.osToK8sIntgPatchPortName())
+ .build();
+ ifaceConfig.addPatchMode(k8sNode.k8sToOsIntgPatchPortName(), k8sIntOsIntPatchDesc);
+
+ // external bridge -> router bridge
+ PatchDescription extRouterPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(k8sNode.extBridgeName())
+ .ifaceName(k8sNode.extToRouterPatchPortName())
+ .peer(k8sNode.routerToExtPatchPortName())
+ .build();
+ ifaceConfig.addPatchMode(k8sNode.extToRouterPatchPortName(), extRouterPatchDesc);
}
}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sHostStore.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sHostStore.java
index 73dd7a4..360f2fa 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sHostStore.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sHostStore.java
@@ -21,11 +21,13 @@
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.k8snode.api.DefaultK8sHost;
+import org.onosproject.k8snode.api.K8sBridge;
import org.onosproject.k8snode.api.K8sHost;
import org.onosproject.k8snode.api.K8sHostEvent;
import org.onosproject.k8snode.api.K8sHostState;
import org.onosproject.k8snode.api.K8sHostStore;
import org.onosproject.k8snode.api.K8sHostStoreDelegate;
+import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.k8snode.api.K8sTunnelBridge;
import org.onosproject.store.AbstractStore;
import org.onosproject.store.serializers.KryoNamespaces;
@@ -81,7 +83,9 @@
.register(K8sHost.class)
.register(DefaultK8sHost.class)
.register(K8sHostState.class)
+ .register(K8sBridge.class)
.register(K8sTunnelBridge.class)
+ .register(K8sRouterBridge.class)
.register(Collection.class)
.build();