Fix: allow to invoke service IP from local k8s node
Change-Id: I8c39703f35319bb51bbd175e7ff717d5e6ec5250
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 5ddc872..d7d7638 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -15,7 +15,9 @@
*/
package org.onosproject.k8snetworking.impl;
+import org.onlab.packet.ARP;
import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpPrefix;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
@@ -31,6 +33,7 @@
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
+import org.onosproject.net.Device;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.DriverService;
@@ -38,6 +41,7 @@
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.net.packet.PacketService;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
@@ -51,11 +55,25 @@
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.B_CLASS;
+import static org.onosproject.k8snetworking.api.Constants.DST;
+import static org.onosproject.k8snetworking.api.Constants.HOST_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.api.Constants.LOCAL_ENTRY_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_ARP_REPLY_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_GATEWAY_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_LOCAL_BRIDGE_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.SHIFTED_IP_PREFIX;
+import static org.onosproject.k8snetworking.api.Constants.SHIFTED_LOCAL_IP_PREFIX;
+import static org.onosproject.k8snetworking.api.Constants.SRC;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.shiftIpDomain;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.tunnelPortNumByNetId;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildLoadExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -67,7 +85,8 @@
private final Logger log = getLogger(getClass());
- private static final int GW_IP_PREFIX = 32;
+ private static final String REQUEST = "req";
+ private static final String REPLY = "rep";
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -128,11 +147,12 @@
}
private void setGatewayRule(K8sNetwork k8sNetwork, boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(), GW_IP_PREFIX));
-
for (K8sNode node : k8sNodeService.completeNodes()) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
if (node.hostname().equals(k8sNetwork.name())) {
@@ -159,9 +179,120 @@
PRIORITY_GATEWAY_RULE,
ROUTING_TABLE,
install);
+
+ if (node.hostname().equals(k8sNetwork.name())) {
+ sBuilder = DefaultTrafficSelector.builder()
+ .matchInPort(PortNumber.LOCAL)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+
+ tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(node.intgToLocalPatchPortNum());
+
+ k8sFlowRuleService.setRule(
+ appId,
+ node.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_LOCAL_BRIDGE_RULE,
+ ROUTING_TABLE,
+ install);
+ }
}
}
+ private void setLocalBridgeRules(K8sNetwork k8sNetwork, boolean install) {
+ for (K8sNode node : k8sNodeService.completeNodes()) {
+ if (node.hostname().equals(k8sNetwork.name())) {
+ setLocalBridgeRule(k8sNetwork, node, REQUEST, install);
+ setLocalBridgeRule(k8sNetwork, node, REPLY, install);
+ }
+ }
+ }
+
+ private void setLocalBridgeRule(K8sNetwork k8sNetwork, K8sNode k8sNode,
+ String type, boolean install) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4);
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ ExtensionTreatment loadTreatment = null;
+
+ if (REQUEST.equals(type)) {
+ loadTreatment = buildLoadExtension(deviceService.getDevice(
+ k8sNode.localBridge()), B_CLASS, SRC, SHIFTED_LOCAL_IP_PREFIX);
+ }
+
+ if (REPLY.equals(type)) {
+ loadTreatment = buildLoadExtension(deviceService.getDevice(
+ k8sNode.localBridge()), B_CLASS, DST, SHIFTED_IP_PREFIX);
+ }
+
+ tBuilder.extension(loadTreatment, k8sNode.localBridge());
+
+ if (REQUEST.equals(type)) {
+ sBuilder.matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+ tBuilder.setOutput(PortNumber.LOCAL);
+ }
+
+ if (REPLY.equals(type)) {
+ sBuilder.matchIPSrc(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+ tBuilder.setOutput(k8sNode.localToIntgPatchPortNumber());
+ }
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.localBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_LOCAL_BRIDGE_RULE,
+ LOCAL_ENTRY_TABLE,
+ install);
+ }
+
+ private void setLocalBridgeArpRules(K8sNetwork k8sNetwork, boolean install) {
+ for (K8sNode node : k8sNodeService.completeNodes()) {
+ if (node.hostname().equals(k8sNetwork.name())) {
+ setLocalBridgeArpRule(k8sNetwork, node, install);
+ }
+ }
+ }
+
+ private void setLocalBridgeArpRule(K8sNetwork k8sNetwork, K8sNode k8sNode, boolean install) {
+ Device device = deviceService.getDevice(k8sNode.localBridge());
+
+ String shiftedLocalIp = shiftIpDomain(
+ k8sNetwork.gatewayIp().toString(), SHIFTED_LOCAL_IP_PREFIX);
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_ARP)
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(Ip4Address.valueOf(shiftedLocalIp))
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setArpOp(ARP.OP_REPLY)
+ .extension(buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(buildMoveArpShaToThaExtension(device), device.id())
+ .extension(buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpSpa(Ip4Address.valueOf(shiftedLocalIp))
+ .setArpSha(k8sNode.intgBridgeMac())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ device.id(),
+ selector,
+ treatment,
+ PRIORITY_ARP_REPLY_RULE,
+ LOCAL_ENTRY_TABLE,
+ install);
+ }
+
private class InternalK8sNetworkListener implements K8sNetworkListener {
private boolean isRelevantHelper() {
@@ -189,6 +320,8 @@
}
setGatewayRule(event.subject(), true);
+ setLocalBridgeRules(event.subject(), true);
+ setLocalBridgeArpRules(event.subject(), true);
}
private void processNetworkRemoval(K8sNetworkEvent event) {
@@ -197,6 +330,8 @@
}
setGatewayRule(event.subject(), false);
+ setLocalBridgeRules(event.subject(), false);
+ setLocalBridgeArpRules(event.subject(), false);
}
}
@@ -225,6 +360,8 @@
}
k8sNetworkService.networks().forEach(n -> setGatewayRule(n, true));
+ k8sNetworkService.networks().forEach(n -> setLocalBridgeRules(n, true));
+ k8sNetworkService.networks().forEach(n -> setLocalBridgeArpRules(n, true));
}
}
}