Support NodePort communication model at k8s passthrough mode
Change-Id: I2179ebc9a4812493619c56aa270d8fc4821efbb2
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
index 0a549cf..c87dfe7 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
@@ -67,17 +67,15 @@
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.NODE_IP_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_CIDR_RULE;
-import static org.onosproject.k8snetworking.api.Constants.PRIORITY_NODE_PORT_INTER_RULE;
-import static org.onosproject.k8snetworking.api.Constants.PRIORITY_NODE_PORT_REMOTE_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_INTER_ROUTING_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_NODE_PORT_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
import static org.onosproject.k8snetworking.api.Constants.SRC;
+import static org.onosproject.k8snetworking.api.Constants.TUN_ENTRY_TABLE;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.getBclassIpPrefixFromCidr;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.getPropertyValue;
-import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.tunnelPortNumByNetId;
-import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.unshiftIpDomain;
-import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildLoadExtension;
+import static org.onosproject.k8snode.api.K8sApiConfig.Mode.PASSTHROUGH;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -94,8 +92,6 @@
private static final int HOST_CIDR = 32;
private static final String SERVICE_CIDR = "serviceCidr";
private static final String B_CLASS_SUFFIX = "0.0/16";
- private static final String C_CLASS_SUFFIX = ".0/24";
-
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -167,120 +163,68 @@
String clusterIp = service.getSpec().getClusterIP();
for (ServicePort servicePort : service.getSpec().getPorts()) {
setNodeToServiceRules(k8sNode, clusterIp, servicePort, install);
- setServiceToNodeLocalRules(k8sNode, clusterIp, servicePort, install);
- setServiceToNodeRemoteRules(k8sNode, clusterIp, servicePort, install);
- setExtToIngrRules(k8sNode, servicePort, install);
+ setServiceToNodeRules(k8sNode, clusterIp, servicePort, install);
}
+ }
+ private void setIntgToExtRules(K8sNode k8sNode, String serviceCidr,
+ boolean install) {
+ // for local traffic, we add default flow rules for steering traffic from
+ // integration bridge to external bridge through patch port
+ // for remote traffic, we add default flow rules for steering traffic from
+ // integration bridge to tun bridge through patch port
k8sNodeService.completeNodes().forEach(n -> {
String podCidr = k8sNetworkService.network(n.hostname()).cidr();
String fullCidr = NODE_IP_PREFIX + "." +
podCidr.split("\\.")[2] + "." + B_CLASS_SUFFIX;
- if (n.equals(k8sNode)) {
- setIntgToExtLocalRules(k8sNode, getServiceCidr(), fullCidr, install);
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(IpPrefix.valueOf(serviceCidr))
+ .matchIPDst(IpPrefix.valueOf(fullCidr));
+
+ PortNumber output;
+ if (n.hostname().equals(k8sNode.hostname())) {
+ output = k8sNode.intgToExtPatchPortNum();
} else {
- setIntgToExtRemoteRules(k8sNode, n, getServiceCidr(), fullCidr, install);
+ output = k8sNode.intgToTunPortNum();
}
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(output);
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_CIDR_RULE,
+ ROUTING_TABLE,
+ install);
});
-
- setDefaultExtEgrRule(k8sNode, install);
}
- private void setDefaultExtEgrRule(K8sNode k8sNode, boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchInPort(PortNumber.LOCAL)
- .matchEthType(Ethernet.TYPE_IPV4);
+ private void setTunToIntgRules(K8sNode k8sNode, boolean install) {
+ String podCidr = k8sNetworkService.network(k8sNode.hostname()).cidr();
+ String fullCidr = NODE_IP_PREFIX + "." +
+ podCidr.split("\\.")[2] + "." + B_CLASS_SUFFIX;
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setOutput(k8sNode.extBridgePortNum());
-
- k8sFlowRuleService.setRule(
- appId,
- k8sNode.extBridge(),
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_NODE_PORT_INTER_RULE,
- EXT_ENTRY_TABLE,
- install);
- }
-
- private void setIntgToExtLocalRules(K8sNode k8sNode, String serviceCidr,
- String shiftedCidr, boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPSrc(IpPrefix.valueOf(serviceCidr))
- .matchIPDst(IpPrefix.valueOf(shiftedCidr));
+ .matchIPDst(IpPrefix.valueOf(fullCidr))
+ .build();
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setOutput(k8sNode.intgToExtPatchPortNum());
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(k8sNode.tunToIntgPortNum())
+ .build();
k8sFlowRuleService.setRule(
appId,
- k8sNode.intgBridge(),
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_CIDR_RULE,
- ROUTING_TABLE,
- install);
- }
-
- private void setIntgToExtRemoteRules(K8sNode k8sNodeLocal, K8sNode k8sNodeRemote,
- String serviceCidr, String shiftedCidr,
- boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPSrc(IpPrefix.valueOf(serviceCidr))
- .matchIPDst(IpPrefix.valueOf(shiftedCidr));
-
- ExtensionTreatment remote = buildExtension(deviceService,
- k8sNodeLocal.intgBridge(), k8sNodeRemote.dataIp().getIp4Address());
-
- PortNumber portNumber = tunnelPortNumByNetId(
- k8sNodeLocal.hostname(), k8sNetworkService, k8sNodeLocal);
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .extension(remote, k8sNodeLocal.intgBridge())
- .setOutput(portNumber);
-
- k8sFlowRuleService.setRule(
- appId,
- k8sNodeLocal.intgBridge(),
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_CIDR_RULE,
- ROUTING_TABLE,
- install);
- }
-
- private void setExtToIngrRules(K8sNode k8sNode, ServicePort servicePort,
- boolean install) {
- String protocol = servicePort.getProtocol();
- int nodePort = servicePort.getNodePort();
- DeviceId deviceId = k8sNode.extBridge();
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), HOST_CIDR));
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setOutput(PortNumber.LOCAL);
-
- if (TCP.equals(protocol)) {
- sBuilder.matchIPProtocol(IPv4.PROTOCOL_TCP)
- .matchTcpSrc(TpPort.tpPort(nodePort));
- } else if (UDP.equals(protocol)) {
- sBuilder.matchIPProtocol(IPv4.PROTOCOL_UDP)
- .matchUdpSrc(TpPort.tpPort(nodePort));
- }
-
- k8sFlowRuleService.setRule(
- appId,
- deviceId,
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_NODE_PORT_RULE,
- EXT_ENTRY_TABLE,
+ k8sNode.tunBridge(),
+ selector,
+ treatment,
+ PRIORITY_INTER_ROUTING_RULE,
+ TUN_ENTRY_TABLE,
install);
}
@@ -295,7 +239,7 @@
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), HOST_CIDR));
+ .matchIPDst(IpPrefix.valueOf(k8sNode.nodeIp(), HOST_CIDR));
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
.setIpDst(IpAddress.valueOf(clusterIp));
@@ -328,86 +272,30 @@
install);
}
- private void setServiceToNodeLocalRules(K8sNode k8sNode,
- String clusterIp,
- ServicePort servicePort,
- boolean install) {
+ private void setServiceToNodeRules(K8sNode k8sNode,
+ String clusterIp,
+ ServicePort servicePort,
+ boolean install) {
String protocol = servicePort.getProtocol();
int nodePort = servicePort.getNodePort();
int svcPort = servicePort.getPort();
DeviceId deviceId = k8sNode.extBridge();
- String extBridgeIp = k8sNode.extBridgeIp().toString();
- String extBridgePrefix = getBclassIpPrefixFromCidr(extBridgeIp);
+ String nodeIp = k8sNode.nodeIp().toString();
+ String nodeIpPrefix = getBclassIpPrefixFromCidr(nodeIp);
- String podCidr = k8sNetworkService.network(k8sNode.hostname()).cidr();
- String nodePrefix = NODE_IP_PREFIX + "." + podCidr.split("\\.")[2];
-
- if (extBridgePrefix == null) {
+ if (nodeIpPrefix == null) {
return;
}
- String shiftedIp = unshiftIpDomain(extBridgeIp, extBridgePrefix, nodePrefix);
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchInPort(k8sNode.extToIntgPatchPortNum())
- .matchIPSrc(IpPrefix.valueOf(IpAddress.valueOf(clusterIp), HOST_CIDR))
- .matchIPDst(IpPrefix.valueOf(IpAddress.valueOf(shiftedIp), HOST_CIDR));
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setIpSrc(k8sNode.extBridgeIp())
- .setEthSrc(k8sNode.extBridgeMac());
-
- if (TCP.equals(protocol)) {
- sBuilder.matchIPProtocol(IPv4.PROTOCOL_TCP)
- .matchTcpSrc(TpPort.tpPort(svcPort));
- tBuilder.setTcpSrc(TpPort.tpPort(nodePort));
- } else if (UDP.equals(protocol)) {
- sBuilder.matchIPProtocol(IPv4.PROTOCOL_UDP)
- .matchUdpSrc(TpPort.tpPort(svcPort));
- tBuilder.setUdpSrc(TpPort.tpPort(nodePort));
- }
-
- String gatewayIp = k8sNode.extGatewayIp().toString();
- String gatewayPrefix = getBclassIpPrefixFromCidr(gatewayIp);
-
- if (gatewayPrefix == null) {
- return;
- }
-
- ExtensionTreatment loadTreatment = buildLoadExtension(
- deviceService.getDevice(deviceId), B_CLASS, DST, gatewayPrefix);
- tBuilder.extension(loadTreatment, deviceId)
- .setOutput(PortNumber.LOCAL);
-
- k8sFlowRuleService.setRule(
- appId,
- deviceId,
- sBuilder.build(),
- tBuilder.build(),
- PRIORITY_NODE_PORT_RULE,
- EXT_ENTRY_TABLE,
- install);
- }
-
- private void setServiceToNodeRemoteRules(K8sNode k8sNode,
- String clusterIp,
- ServicePort servicePort,
- boolean install) {
- String protocol = servicePort.getProtocol();
- int nodePort = servicePort.getNodePort();
- int svcPort = servicePort.getPort();
- DeviceId deviceId = k8sNode.extBridge();
-
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
.matchInPort(k8sNode.extToIntgPatchPortNum())
.matchIPSrc(IpPrefix.valueOf(IpAddress.valueOf(clusterIp), HOST_CIDR));
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setIpSrc(k8sNode.extBridgeIp())
- .setEthSrc(k8sNode.extBridgeMac());
+ .setIpSrc(k8sNode.nodeIp())
+ .setEthSrc(k8sNode.nodeMac());
if (TCP.equals(protocol)) {
sBuilder.matchIPProtocol(IPv4.PROTOCOL_TCP)
@@ -419,24 +307,30 @@
tBuilder.setUdpSrc(TpPort.tpPort(nodePort));
}
- String gatewayIp = k8sNode.extGatewayIp().toString();
- String prefix = getBclassIpPrefixFromCidr(gatewayIp);
-
- if (prefix == null) {
- return;
- }
-
ExtensionTreatment loadTreatment = buildLoadExtension(
- deviceService.getDevice(deviceId), B_CLASS, DST, prefix);
- tBuilder.extension(loadTreatment, deviceId)
- .setOutput(k8sNode.extBridgePortNum());
+ deviceService.getDevice(deviceId), B_CLASS, DST, nodeIpPrefix);
+ tBuilder.extension(loadTreatment, deviceId);
+
+ // in passthrough mode, we steer the traffic to the openstack intg bridge
+ // in normal mode, we steer the traffic to the local port
+ if (k8sNode.mode() == PASSTHROUGH) {
+ PortNumber output = k8sNode.portNumByName(k8sNode.extBridge(),
+ k8sNode.k8sExtToOsPatchPortName());
+ if (output == null) {
+ log.warn("Kubernetes external to OpenStack patch port is null");
+ return;
+ }
+ tBuilder.setOutput(output);
+ } else {
+ tBuilder.setOutput(PortNumber.LOCAL);
+ }
k8sFlowRuleService.setRule(
appId,
deviceId,
sBuilder.build(),
tBuilder.build(),
- PRIORITY_NODE_PORT_REMOTE_RULE,
+ PRIORITY_NODE_PORT_RULE,
EXT_ENTRY_TABLE,
install);
}
@@ -504,6 +398,9 @@
k8sServiceService.services().stream()
.filter(s -> NODE_PORT_TYPE.equals(s.getSpec().getType()))
.forEach(s -> processNodePortEvent(k8sNode, s, true));
+
+ setIntgToExtRules(k8sNode, getServiceCidr(), true);
+ setTunToIntgRules(k8sNode, true);
}
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
index 0c68e20..9e24daa 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
@@ -63,6 +63,7 @@
private static final String K8S_NODE_IP = "k8sNodeIp";
private static final String OS_K8S_INT_PORT_NAME = "osK8sIntPortName";
+ private static final String OS_K8S_EXT_PORT_NAME = "osK8sExtPortName";
private static final String POD_CIDR = "podCidr";
private static final String SERVICE_CIDR = "serviceCidr";
private static final String POD_GW_IP = "podGwIp";
@@ -121,7 +122,8 @@
String srcPodPrefix = getBclassIpPrefixFromCidr(nodePodCidr);
String podCidr = srcPodPrefix + B_CLASS_SUFFIX;
String osK8sIntPortName = k8sNode.osToK8sIntgPatchPortName();
- String k8sIntOsPortMac = k8sNode.portMacByName(k8sNode.k8sIntgToOsPatchPortName()).toString();
+ String k8sIntOsPortMac = k8sNode.portMacByName(k8sNode.intgBridge(),
+ k8sNode.k8sIntgToOsPatchPortName()).toString();
String path = install ? "node/pt-install" : "node/pt-uninstall";
@@ -159,6 +161,43 @@
}
}
+ private void setCniPtNodePortRules(K8sNode k8sNode, boolean install) {
+ String k8sNodeIp = k8sNode.nodeIp().toString();
+ String osK8sExtPortName = k8sNode.osToK8sExtPatchPortName();
+
+ String path = install ? "nodeport/pt-install" : "nodeport/pt-uninstall";
+
+ String jsonString = "";
+
+ try {
+ jsonString = new JSONObject()
+ .put(K8S_NODE_IP, k8sNodeIp)
+ .put(SERVICE_CIDR, SERVICE_IP_CIDR_DEFAULT)
+ .put(OS_K8S_EXT_PORT_NAME, osK8sExtPortName)
+ .toString();
+ log.info("push integration configuration {}", jsonString);
+ } catch (JSONException e) {
+ log.error("Failed to generate JSON string");
+ return;
+ }
+
+ HttpAuthenticationFeature feature =
+ HttpAuthenticationFeature.basic(ONOS_USERNAME, ONOS_PASSWORD);
+
+ final Client client = ClientBuilder.newClient();
+ client.register(feature);
+ String host = "http://" + k8sNode.managementIp().toString() + ":" + ONOS_PORT + "/";
+ String endpoint = host + OS_K8S_INTEGRATION_EP;
+ WebTarget wt = client.target(endpoint).path(path);
+ Response response = wt.request(MediaType.APPLICATION_JSON_TYPE)
+ .put(Entity.json(jsonString));
+ final int status = response.getStatus();
+
+ if (status != 200) {
+ log.error("Failed to install/uninstall openstack k8s CNI PT node port rules.");
+ }
+ }
+
private class InternalK8sNodeListener implements K8sNodeListener {
@Override
@@ -190,6 +229,7 @@
}
setCniPtNodeRules(k8sNode, true);
+ setCniPtNodePortRules(k8sNode, true);
}
private void processNodeIncompletion(K8sNode k8sNode) {
@@ -198,6 +238,7 @@
}
setCniPtNodeRules(k8sNode, false);
+ setCniPtNodePortRules(k8sNode, false);
}
}
}
\ No newline at end of file
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingArpHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingArpHandler.java
index 93bde9b..1514f6c 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingArpHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingArpHandler.java
@@ -178,7 +178,6 @@
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_ARP)
.matchArpOp(ARP.OP_REPLY)
- .matchArpSpa(Ip4Address.valueOf(k8sNode.extGatewayIp().toString()))
.build();
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
index d5c047f..9780435 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
@@ -15,11 +15,13 @@
*/
package org.onosproject.k8snetworking.impl;
+import org.apache.commons.net.util.SubnetUtils;
import org.onlab.packet.ARP;
import org.onlab.packet.EthType;
import org.onlab.packet.Ethernet;
import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
import org.onlab.packet.MacAddress;
import org.onlab.packet.VlanId;
import org.onlab.util.KryoNamespace;
@@ -38,9 +40,10 @@
import org.onosproject.k8snetworking.api.K8sServiceService;
import org.onosproject.k8snode.api.K8sHostService;
import org.onosproject.k8snode.api.K8sNode;
+import org.onosproject.k8snode.api.K8sNodeAdminService;
import org.onosproject.k8snode.api.K8sNodeEvent;
+import org.onosproject.k8snode.api.K8sNodeInfo;
import org.onosproject.k8snode.api.K8sNodeListener;
-import org.onosproject.k8snode.api.K8sNodeService;
import org.onosproject.mastership.MastershipService;
import org.onosproject.net.ConnectPoint;
import org.onosproject.net.DeviceId;
@@ -76,11 +79,14 @@
import java.util.stream.Collectors;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.packet.IpAddress.Version.INET;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.k8snetworking.api.Constants.ARP_BROADCAST_MODE;
import static org.onosproject.k8snetworking.api.Constants.ARP_PROXY_MODE;
import static org.onosproject.k8snetworking.api.Constants.ARP_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.api.Constants.NODE_FAKE_IP_STR;
+import static org.onosproject.k8snetworking.api.Constants.NODE_FAKE_MAC_STR;
import static org.onosproject.k8snetworking.api.Constants.NODE_IP_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_ARP_CONTROL_RULE;
import static org.onosproject.k8snetworking.api.Constants.SERVICE_FAKE_MAC_STR;
@@ -90,6 +96,7 @@
import static org.onosproject.k8snetworking.impl.OsgiPropertyConstants.GATEWAY_MAC;
import static org.onosproject.k8snetworking.impl.OsgiPropertyConstants.GATEWAY_MAC_DEFAULT;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.allK8sDevices;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.getGatewayIp;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.getPropertyValue;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.unshiftIpDomain;
@@ -139,7 +146,7 @@
protected StorageService storageService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
- protected K8sNodeService k8sNodeService;
+ protected K8sNodeAdminService k8sNodeService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected K8sHostService k8sHostService;
@@ -159,8 +166,6 @@
/** ARP processing mode, broadcast | proxy (default). */
protected String arpMode = ARP_MODE_DEFAULT;
- private MacAddress gwMacAddress;
-
private ConsistentMap<IpAddress, MacAddress> extHostMacStore;
private final ExecutorService eventExecutor = newSingleThreadExecutor(
@@ -305,20 +310,23 @@
String targetIpPrefix = targetIp.toString().split("\\.")[1];
String nodePrefix = NODE_IP_PREFIX + "." + targetIpPrefix;
- String exBridgeCidr = k8sNodeService.completeNodes().stream()
- .map(n -> n.extBridgeIp().toString()).findAny().orElse(null);
+ String origNodeCidr = k8sNodeService.completeNodes().stream()
+ .map(n -> n.nodeIp().toString()).findAny().orElse(null);
- if (exBridgeCidr != null) {
- String extBridgeIp = unshiftIpDomain(targetIp.toString(),
- nodePrefix, exBridgeCidr);
+ if (origNodeCidr != null) {
+ String origNodeIp = unshiftIpDomain(targetIp.toString(),
+ nodePrefix, origNodeCidr);
+ IpPrefix k8sNodeIpCidr = IpPrefix.valueOf(IpAddress.valueOf(origNodeCidr), 24);
+ SubnetUtils k8sNodeSubnet = new SubnetUtils(k8sNodeIpCidr.toString());
+ String k8sNodeGateway = getGatewayIp(k8sNodeIpCidr.toString()).toString();
+ String seekIp = "";
- replyMac = k8sNodeService.completeNodes().stream()
- .filter(n -> extBridgeIp.equals(n.extBridgeIp().toString()))
- .map(K8sNode::extBridgeMac).findAny().orElse(null);
-
- if (replyMac == null) {
- replyMac = extHostMacStore.asJavaMap().get(
- IpAddress.valueOf(extBridgeIp));
+ if (!k8sNodeSubnet.getInfo().isInRange(origNodeIp)) {
+ replyMac = extHostMacStore.asJavaMap().get(IpAddress.valueOf(k8sNodeGateway));
+ seekIp = k8sNodeGateway;
+ } else {
+ replyMac = extHostMacStore.asJavaMap().get(IpAddress.valueOf(origNodeIp));
+ seekIp = origNodeIp;
}
// if the source hosts are not in k8s cluster range,
@@ -328,9 +336,11 @@
K8sNode k8sNode = k8sNodeService.node(cp.deviceId());
if (k8sNode != null) {
- setArpRequest(k8sNode.extBridgeMac().toBytes(),
- k8sNode.extBridgeIp().toOctets(),
- IpAddress.valueOf(extBridgeIp).toOctets(),
+ // we use fake IP and MAC address as a source to
+ // query destination MAC address
+ setArpRequest(MacAddress.valueOf(NODE_FAKE_MAC_STR).toBytes(),
+ IpAddress.valueOf(NODE_FAKE_IP_STR).toOctets(),
+ IpAddress.valueOf(seekIp).toOctets(),
k8sNode);
context.block();
return;
@@ -363,20 +373,34 @@
private void processArpReply(PacketContext context, Ethernet ethPacket) {
ARP arpPacket = (ARP) ethPacket.getPayload();
- ConnectPoint cp = context.inPacket().receivedFrom();
- K8sNode k8sNode = k8sNodeService.node(cp.deviceId());
- if (k8sNode != null &&
- ethPacket.getDestinationMAC().equals(k8sNode.extBridgeMac())) {
- IpAddress srcIp = IpAddress.valueOf(IpAddress.Version.INET,
- arpPacket.getSenderProtocolAddress());
- MacAddress srcMac = MacAddress.valueOf(arpPacket.getSenderHardwareAddress());
+ IpAddress srcIp = IpAddress.valueOf(INET, arpPacket.getSenderProtocolAddress());
+ MacAddress srcMac = MacAddress.valueOf(arpPacket.getSenderHardwareAddress());
+ IpAddress dstIp = IpAddress.valueOf(INET, arpPacket.getTargetProtocolAddress());
+ if (dstIp.equals(IpAddress.valueOf(NODE_FAKE_IP_STR))) {
// we only add the host IP - MAC map store once,
// mutable MAP scenario is not considered for now
if (!extHostMacStore.containsKey(srcIp)) {
extHostMacStore.put(srcIp, srcMac);
}
+
+ K8sNode k8sNode = k8sNodeService.nodes().stream()
+ .filter(n -> n.nodeIp().equals(srcIp))
+ .findAny().orElse(null);
+
+ if (k8sNode == null) {
+ return;
+ } else {
+ if (k8sNode.nodeInfo().nodeMac() != null) {
+ return;
+ }
+ }
+
+ // we update node MAC address which will be referred in node port scenario
+ K8sNodeInfo nodeInfo = new K8sNodeInfo(k8sNode.nodeIp(), srcMac);
+ K8sNode updatedNode = k8sNode.updateNodeInfo(nodeInfo);
+ k8sNodeService.updateNode(updatedNode);
}
}
@@ -385,12 +409,20 @@
Ethernet ethRequest = ARP.buildArpRequest(senderMac,
senderIp, targetIp, VlanId.NO_VID);
+ // TODO: we need to find a way of sending out ARP request to learn
+ // MAC addresses in NORMAL mode
+ PortNumber k8sExtToOsPatchPort = k8sNode.portNumByName(k8sNode.extBridge(),
+ k8sNode.k8sExtToOsPatchPortName());
+ if (k8sExtToOsPatchPort == null) {
+ log.warn("Kubernetes external to OpenStack patch port is null");
+ return;
+ }
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- .setOutput(k8sNode.intgToExtPatchPortNum())
+ .setOutput(k8sExtToOsPatchPort)
.build();
packetService.emit(new DefaultOutboundPacket(
- k8sNode.intgBridge(),
+ k8sNode.extBridge(),
treatment,
ByteBuffer.wrap(ethRequest.serialize())));
}
@@ -466,6 +498,7 @@
}
setDefaultArpRule(node, true);
+ learnK8sNodeMac(node);
}
private void processNodeIncompletion(K8sNode node) {
@@ -516,5 +549,17 @@
install
);
}
+
+ private void learnK8sNodeMac(K8sNode k8sNode) {
+ // if we already have a learned MAC address, we skip learning process
+ if (k8sNode.nodeMac() != null) {
+ return;
+ }
+
+ setArpRequest(MacAddress.valueOf(NODE_FAKE_MAC_STR).toBytes(),
+ IpAddress.valueOf(NODE_FAKE_IP_STR).toOctets(),
+ k8sNode.nodeIp().toOctets(),
+ k8sNode);
+ }
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 8083e88..2c580cb 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -64,6 +64,7 @@
import static org.onosproject.k8snetworking.api.Constants.HOST_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.LOCAL_ENTRY_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.NODE_IP_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_ARP_REPLY_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_GATEWAY_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_INTER_NODE_RULE;
@@ -283,6 +284,23 @@
PRIORITY_INTER_NODE_RULE,
TUN_ENTRY_TABLE,
install);
+
+ String nodeIpPrefix = NODE_IP_PREFIX + ".0.0.0/8";
+
+ TrafficSelector nodePortSelector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(IpPrefix.valueOf(nodeIpPrefix))
+ .matchIPDst(IpPrefix.valueOf(dstNode.podCidr()))
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ dstNode.tunBridge(),
+ nodePortSelector,
+ treatment,
+ PRIORITY_INTER_NODE_RULE,
+ TUN_ENTRY_TABLE,
+ install);
}
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
index 938629f..f691b1f 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
@@ -295,6 +295,19 @@
}
/**
+ * Obtains gateway IP address of the given subnet.
+ *
+ * @param cidr CIDR
+ * @return gateway IP address
+ */
+ public static IpAddress getGatewayIp(String cidr) {
+ SubnetUtils utils = new SubnetUtils(cidr);
+ utils.setInclusiveHostCount(false);
+ SubnetUtils.SubnetInfo info = utils.getInfo();
+ return IpAddress.valueOf(info.getLowAddress());
+ }
+
+ /**
* Obtains flow group key from the given id.
*
* @param groupId flow group identifier