Support NodePort communication model at k8s passthrough mode
Change-Id: I2179ebc9a4812493619c56aa270d8fc4821efbb2
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
index 0a549cf..c87dfe7 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
@@ -67,17 +67,15 @@
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.NODE_IP_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_CIDR_RULE;
-import static org.onosproject.k8snetworking.api.Constants.PRIORITY_NODE_PORT_INTER_RULE;
-import static org.onosproject.k8snetworking.api.Constants.PRIORITY_NODE_PORT_REMOTE_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_INTER_ROUTING_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_NODE_PORT_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
import static org.onosproject.k8snetworking.api.Constants.SRC;
+import static org.onosproject.k8snetworking.api.Constants.TUN_ENTRY_TABLE;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.getBclassIpPrefixFromCidr;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.getPropertyValue;
-import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.tunnelPortNumByNetId;
-import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.unshiftIpDomain;
-import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildLoadExtension;
+import static org.onosproject.k8snode.api.K8sApiConfig.Mode.PASSTHROUGH;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -94,8 +92,6 @@
private static final int HOST_CIDR = 32;
private static final String SERVICE_CIDR = "serviceCidr";
private static final String B_CLASS_SUFFIX = "0.0/16";
- private static final String C_CLASS_SUFFIX = ".0/24";
-
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -167,120 +163,68 @@
String clusterIp = service.getSpec().getClusterIP();
for (ServicePort servicePort : service.getSpec().getPorts()) {
setNodeToServiceRules(k8sNode, clusterIp, servicePort, install);
- setServiceToNodeLocalRules(k8sNode, clusterIp, servicePort, install);
- setServiceToNodeRemoteRules(k8sNode, clusterIp, servicePort, install);
- setExtToIngrRules(k8sNode, servicePort, install);
+ setServiceToNodeRules(k8sNode, clusterIp, servicePort, install);
}
+ }
+ private void setIntgToExtRules(K8sNode k8sNode, String serviceCidr,
+ boolean install) {
+ // for local traffic, we add default flow rules for steering traffic from
+ // integration bridge to external bridge through patch port
+ // for remote traffic, we add default flow rules for steering traffic from
+ // integration bridge to tun bridge through patch port
k8sNodeService.completeNodes().forEach(n -> {
String podCidr = k8sNetworkService.network(n.hostname()).cidr();
String fullCidr = NODE_IP_PREFIX + "." +
podCidr.split("\\.")[2] + "." + B_CLASS_SUFFIX;
- if (n.equals(k8sNode)) {
- setIntgToExtLocalRules(k8sNode, getServiceCidr(), fullCidr, install);
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(IpPrefix.valueOf(serviceCidr))
+ .matchIPDst(IpPrefix.valueOf(fullCidr));
+
+ PortNumber output;
+ if (n.hostname().equals(k8sNode.hostname())) {
+ output = k8sNode.intgToExtPatchPortNum();
} else {
- setIntgToExtRemoteRules(k8sNode, n, getServiceCidr(), fullCidr, install);
+ output = k8sNode.intgToTunPortNum();
}
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(output);
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_CIDR_RULE,
+ ROUTING_TABLE,
+ install);
});
-
- setDefaultExtEgrRule(k8sNode, install);
}
- private void setDefaultExtEgrRule(K8sNode k8sNode, boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchInPort(PortNumber.LOCAL)
- .matchEthType(Ethernet.TYPE_IPV4);
+ private void setTunToIntgRules(K8sNode k8sNode, boolean install) {
+ String podCidr = k8sNetworkService.network(k8sNode.hostname()).cidr();
+ String fullCidr = NODE_IP_PREFIX + "." +
+ podCidr.split("\\.")[2] + "." + B_CLASS_SUFFIX;
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setOutput(k8sNode.extBridgePortNum());
-
- k8sFlowRuleService.setRule(
- appId,
- k8sNode.extBridge(),
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_NODE_PORT_INTER_RULE,
- EXT_ENTRY_TABLE,
- install);
- }
-
- private void setIntgToExtLocalRules(K8sNode k8sNode, String serviceCidr,
- String shiftedCidr, boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPSrc(IpPrefix.valueOf(serviceCidr))
- .matchIPDst(IpPrefix.valueOf(shiftedCidr));
+ .matchIPDst(IpPrefix.valueOf(fullCidr))
+ .build();
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setOutput(k8sNode.intgToExtPatchPortNum());
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(k8sNode.tunToIntgPortNum())
+ .build();
k8sFlowRuleService.setRule(
appId,
- k8sNode.intgBridge(),
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_CIDR_RULE,
- ROUTING_TABLE,
- install);
- }
-
- private void setIntgToExtRemoteRules(K8sNode k8sNodeLocal, K8sNode k8sNodeRemote,
- String serviceCidr, String shiftedCidr,
- boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPSrc(IpPrefix.valueOf(serviceCidr))
- .matchIPDst(IpPrefix.valueOf(shiftedCidr));
-
- ExtensionTreatment remote = buildExtension(deviceService,
- k8sNodeLocal.intgBridge(), k8sNodeRemote.dataIp().getIp4Address());
-
- PortNumber portNumber = tunnelPortNumByNetId(
- k8sNodeLocal.hostname(), k8sNetworkService, k8sNodeLocal);
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .extension(remote, k8sNodeLocal.intgBridge())
- .setOutput(portNumber);
-
- k8sFlowRuleService.setRule(
- appId,
- k8sNodeLocal.intgBridge(),
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_CIDR_RULE,
- ROUTING_TABLE,
- install);
- }
-
- private void setExtToIngrRules(K8sNode k8sNode, ServicePort servicePort,
- boolean install) {
- String protocol = servicePort.getProtocol();
- int nodePort = servicePort.getNodePort();
- DeviceId deviceId = k8sNode.extBridge();
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), HOST_CIDR));
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setOutput(PortNumber.LOCAL);
-
- if (TCP.equals(protocol)) {
- sBuilder.matchIPProtocol(IPv4.PROTOCOL_TCP)
- .matchTcpSrc(TpPort.tpPort(nodePort));
- } else if (UDP.equals(protocol)) {
- sBuilder.matchIPProtocol(IPv4.PROTOCOL_UDP)
- .matchUdpSrc(TpPort.tpPort(nodePort));
- }
-
- k8sFlowRuleService.setRule(
- appId,
- deviceId,
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_NODE_PORT_RULE,
- EXT_ENTRY_TABLE,
+ k8sNode.tunBridge(),
+ selector,
+ treatment,
+ PRIORITY_INTER_ROUTING_RULE,
+ TUN_ENTRY_TABLE,
install);
}
@@ -295,7 +239,7 @@
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), HOST_CIDR));
+ .matchIPDst(IpPrefix.valueOf(k8sNode.nodeIp(), HOST_CIDR));
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
.setIpDst(IpAddress.valueOf(clusterIp));
@@ -328,86 +272,30 @@
install);
}
- private void setServiceToNodeLocalRules(K8sNode k8sNode,
- String clusterIp,
- ServicePort servicePort,
- boolean install) {
+ private void setServiceToNodeRules(K8sNode k8sNode,
+ String clusterIp,
+ ServicePort servicePort,
+ boolean install) {
String protocol = servicePort.getProtocol();
int nodePort = servicePort.getNodePort();
int svcPort = servicePort.getPort();
DeviceId deviceId = k8sNode.extBridge();
- String extBridgeIp = k8sNode.extBridgeIp().toString();
- String extBridgePrefix = getBclassIpPrefixFromCidr(extBridgeIp);
+ String nodeIp = k8sNode.nodeIp().toString();
+ String nodeIpPrefix = getBclassIpPrefixFromCidr(nodeIp);
- String podCidr = k8sNetworkService.network(k8sNode.hostname()).cidr();
- String nodePrefix = NODE_IP_PREFIX + "." + podCidr.split("\\.")[2];
-
- if (extBridgePrefix == null) {
+ if (nodeIpPrefix == null) {
return;
}
- String shiftedIp = unshiftIpDomain(extBridgeIp, extBridgePrefix, nodePrefix);
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchInPort(k8sNode.extToIntgPatchPortNum())
- .matchIPSrc(IpPrefix.valueOf(IpAddress.valueOf(clusterIp), HOST_CIDR))
- .matchIPDst(IpPrefix.valueOf(IpAddress.valueOf(shiftedIp), HOST_CIDR));
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setIpSrc(k8sNode.extBridgeIp())
- .setEthSrc(k8sNode.extBridgeMac());
-
- if (TCP.equals(protocol)) {
- sBuilder.matchIPProtocol(IPv4.PROTOCOL_TCP)
- .matchTcpSrc(TpPort.tpPort(svcPort));
- tBuilder.setTcpSrc(TpPort.tpPort(nodePort));
- } else if (UDP.equals(protocol)) {
- sBuilder.matchIPProtocol(IPv4.PROTOCOL_UDP)
- .matchUdpSrc(TpPort.tpPort(svcPort));
- tBuilder.setUdpSrc(TpPort.tpPort(nodePort));
- }
-
- String gatewayIp = k8sNode.extGatewayIp().toString();
- String gatewayPrefix = getBclassIpPrefixFromCidr(gatewayIp);
-
- if (gatewayPrefix == null) {
- return;
- }
-
- ExtensionTreatment loadTreatment = buildLoadExtension(
- deviceService.getDevice(deviceId), B_CLASS, DST, gatewayPrefix);
- tBuilder.extension(loadTreatment, deviceId)
- .setOutput(PortNumber.LOCAL);
-
- k8sFlowRuleService.setRule(
- appId,
- deviceId,
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_NODE_PORT_RULE,
- EXT_ENTRY_TABLE,
- install);
- }
-
- private void setServiceToNodeRemoteRules(K8sNode k8sNode,
- String clusterIp,
- ServicePort servicePort,
- boolean install) {
- String protocol = servicePort.getProtocol();
- int nodePort = servicePort.getNodePort();
- int svcPort = servicePort.getPort();
- DeviceId deviceId = k8sNode.extBridge();
-
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
.matchInPort(k8sNode.extToIntgPatchPortNum())
.matchIPSrc(IpPrefix.valueOf(IpAddress.valueOf(clusterIp), HOST_CIDR));
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setIpSrc(k8sNode.extBridgeIp())
- .setEthSrc(k8sNode.extBridgeMac());
+ .setIpSrc(k8sNode.nodeIp())
+ .setEthSrc(k8sNode.nodeMac());
if (TCP.equals(protocol)) {
sBuilder.matchIPProtocol(IPv4.PROTOCOL_TCP)
@@ -419,24 +307,30 @@
tBuilder.setUdpSrc(TpPort.tpPort(nodePort));
}
- String gatewayIp = k8sNode.extGatewayIp().toString();
- String prefix = getBclassIpPrefixFromCidr(gatewayIp);
-
- if (prefix == null) {
- return;
- }
-
ExtensionTreatment loadTreatment = buildLoadExtension(
- deviceService.getDevice(deviceId), B_CLASS, DST, prefix);
- tBuilder.extension(loadTreatment, deviceId)
- .setOutput(k8sNode.extBridgePortNum());
+ deviceService.getDevice(deviceId), B_CLASS, DST, nodeIpPrefix);
+ tBuilder.extension(loadTreatment, deviceId);
+
+ // in passthrough mode, we steer the traffic to the openstack intg bridge
+ // in normal mode, we steer the traffic to the local port
+ if (k8sNode.mode() == PASSTHROUGH) {
+ PortNumber output = k8sNode.portNumByName(k8sNode.extBridge(),
+ k8sNode.k8sExtToOsPatchPortName());
+ if (output == null) {
+ log.warn("Kubernetes external to OpenStack patch port is null");
+ return;
+ }
+ tBuilder.setOutput(output);
+ } else {
+ tBuilder.setOutput(PortNumber.LOCAL);
+ }
k8sFlowRuleService.setRule(
appId,
deviceId,
sBuilder.build(),
tBuilder.build(),
- PRIORITY_NODE_PORT_REMOTE_RULE,
+ PRIORITY_NODE_PORT_RULE,
EXT_ENTRY_TABLE,
install);
}
@@ -504,6 +398,9 @@
k8sServiceService.services().stream()
.filter(s -> NODE_PORT_TYPE.equals(s.getSpec().getType()))
.forEach(s -> processNodePortEvent(k8sNode, s, true));
+
+ setIntgToExtRules(k8sNode, getServiceCidr(), true);
+ setTunToIntgRules(k8sNode, true);
}
}
}