Fix: allow to invoke service IP from local k8s node
Change-Id: I8c39703f35319bb51bbd175e7ff717d5e6ec5250
diff --git a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
index 2630c30..bca34f2 100644
--- a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
+++ b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
@@ -45,6 +45,7 @@
public static final String SHIFTED_IP_CIDR = "172.10.0.0/16";
public static final String SHIFTED_IP_PREFIX = "172.10";
+ public static final String SHIFTED_LOCAL_IP_PREFIX = "172.11";
public static final String NODE_IP_PREFIX = "182";
public static final String SRC = "src";
@@ -64,6 +65,8 @@
public static final int DEFAULT_SEGMENT_ID = 100;
+ public static final int HOST_PREFIX = 32;
+
// flow priority
public static final int PRIORITY_SNAT_RULE = 26000;
public static final int PRIORITY_TUNNEL_TAG_RULE = 30000;
@@ -73,15 +76,13 @@
public static final int PRIORITY_CT_RULE = 32000;
public static final int PRIORITY_CT_DROP_RULE = 32500;
public static final int PRIORITY_NAT_RULE = 30000;
- public static final int PRIORITY_SERVICE_CIDR_RULE = 32000;
- public static final int PRIORITY_GATEWAY_RULE = 30000;
+ public static final int PRIORITY_GATEWAY_RULE = 31000;
+ public static final int PRIORITY_LOCAL_BRIDGE_RULE = 32000;
public static final int PRIORITY_SWITCHING_RULE = 30000;
public static final int PRIORITY_CIDR_RULE = 30000;
public static final int PRIORITY_NAMESPACE_RULE = 31000;
public static final int PRIORITY_STATEFUL_SNAT_RULE = 41000;
public static final int PRIORITY_EXTERNAL_ROUTING_RULE = 25000;
- public static final int PRIORITY_ARP_GATEWAY_RULE = 41000;
- public static final int PRIORITY_ARP_SUBNET_RULE = 40000;
public static final int PRIORITY_ARP_CONTROL_RULE = 40000;
public static final int PRIORITY_ARP_REPLY_RULE = 40000;
public static final int PRIORITY_ARP_POD_RULE = 39000;
@@ -118,4 +119,6 @@
public static final int EXT_ENTRY_TABLE = 0;
public static final int POD_RESOLUTION_TABLE = 11;
+
+ public static final int LOCAL_ENTRY_TABLE = 0;
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
index 25d8a93..73ce2ef 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
@@ -61,6 +61,7 @@
import static org.onosproject.k8snetworking.api.Constants.DEFAULT_GATEWAY_MAC;
import static org.onosproject.k8snetworking.api.Constants.FORWARDING_TABLE;
import static org.onosproject.k8snetworking.api.Constants.GROUPING_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.HOST_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.JUMP_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.NAMESPACE_TABLE;
@@ -343,7 +344,7 @@
private void setupHostRoutingRule(K8sNetwork k8sNetwork) {
setAnyRoutingRule(IpPrefix.valueOf(
- k8sNetwork.gatewayIp(), 32), null, k8sNetwork);
+ k8sNetwork.gatewayIp(), HOST_PREFIX), null, k8sNetwork);
}
private class InternalK8sNodeListener implements K8sNodeListener {
@@ -391,8 +392,8 @@
deviceEventExecutor.execute(() -> processNetworkCreation(event.subject()));
break;
case K8S_NETWORK_REMOVED:
- break;
default:
+ // do nothing
break;
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 5ddc872..d7d7638 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -15,7 +15,9 @@
*/
package org.onosproject.k8snetworking.impl;
+import org.onlab.packet.ARP;
import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpPrefix;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
@@ -31,6 +33,7 @@
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
+import org.onosproject.net.Device;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.DriverService;
@@ -38,6 +41,7 @@
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.net.packet.PacketService;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
@@ -51,11 +55,25 @@
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.B_CLASS;
+import static org.onosproject.k8snetworking.api.Constants.DST;
+import static org.onosproject.k8snetworking.api.Constants.HOST_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.api.Constants.LOCAL_ENTRY_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_ARP_REPLY_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_GATEWAY_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_LOCAL_BRIDGE_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.SHIFTED_IP_PREFIX;
+import static org.onosproject.k8snetworking.api.Constants.SHIFTED_LOCAL_IP_PREFIX;
+import static org.onosproject.k8snetworking.api.Constants.SRC;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.shiftIpDomain;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.tunnelPortNumByNetId;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildLoadExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -67,7 +85,8 @@
private final Logger log = getLogger(getClass());
- private static final int GW_IP_PREFIX = 32;
+ private static final String REQUEST = "req";
+ private static final String REPLY = "rep";
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -128,11 +147,12 @@
}
private void setGatewayRule(K8sNetwork k8sNetwork, boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(), GW_IP_PREFIX));
-
for (K8sNode node : k8sNodeService.completeNodes()) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
if (node.hostname().equals(k8sNetwork.name())) {
@@ -159,9 +179,120 @@
PRIORITY_GATEWAY_RULE,
ROUTING_TABLE,
install);
+
+ if (node.hostname().equals(k8sNetwork.name())) {
+ sBuilder = DefaultTrafficSelector.builder()
+ .matchInPort(PortNumber.LOCAL)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+
+ tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(node.intgToLocalPatchPortNum());
+
+ k8sFlowRuleService.setRule(
+ appId,
+ node.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_LOCAL_BRIDGE_RULE,
+ ROUTING_TABLE,
+ install);
+ }
}
}
+ private void setLocalBridgeRules(K8sNetwork k8sNetwork, boolean install) {
+ for (K8sNode node : k8sNodeService.completeNodes()) {
+ if (node.hostname().equals(k8sNetwork.name())) {
+ setLocalBridgeRule(k8sNetwork, node, REQUEST, install);
+ setLocalBridgeRule(k8sNetwork, node, REPLY, install);
+ }
+ }
+ }
+
+ private void setLocalBridgeRule(K8sNetwork k8sNetwork, K8sNode k8sNode,
+ String type, boolean install) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4);
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ ExtensionTreatment loadTreatment = null;
+
+ if (REQUEST.equals(type)) {
+ loadTreatment = buildLoadExtension(deviceService.getDevice(
+ k8sNode.localBridge()), B_CLASS, SRC, SHIFTED_LOCAL_IP_PREFIX);
+ }
+
+ if (REPLY.equals(type)) {
+ loadTreatment = buildLoadExtension(deviceService.getDevice(
+ k8sNode.localBridge()), B_CLASS, DST, SHIFTED_IP_PREFIX);
+ }
+
+ tBuilder.extension(loadTreatment, k8sNode.localBridge());
+
+ if (REQUEST.equals(type)) {
+ sBuilder.matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+ tBuilder.setOutput(PortNumber.LOCAL);
+ }
+
+ if (REPLY.equals(type)) {
+ sBuilder.matchIPSrc(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+ tBuilder.setOutput(k8sNode.localToIntgPatchPortNumber());
+ }
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.localBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_LOCAL_BRIDGE_RULE,
+ LOCAL_ENTRY_TABLE,
+ install);
+ }
+
+ private void setLocalBridgeArpRules(K8sNetwork k8sNetwork, boolean install) {
+ for (K8sNode node : k8sNodeService.completeNodes()) {
+ if (node.hostname().equals(k8sNetwork.name())) {
+ setLocalBridgeArpRule(k8sNetwork, node, install);
+ }
+ }
+ }
+
+ private void setLocalBridgeArpRule(K8sNetwork k8sNetwork, K8sNode k8sNode, boolean install) {
+ Device device = deviceService.getDevice(k8sNode.localBridge());
+
+ String shiftedLocalIp = shiftIpDomain(
+ k8sNetwork.gatewayIp().toString(), SHIFTED_LOCAL_IP_PREFIX);
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_ARP)
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(Ip4Address.valueOf(shiftedLocalIp))
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setArpOp(ARP.OP_REPLY)
+ .extension(buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(buildMoveArpShaToThaExtension(device), device.id())
+ .extension(buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpSpa(Ip4Address.valueOf(shiftedLocalIp))
+ .setArpSha(k8sNode.intgBridgeMac())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ device.id(),
+ selector,
+ treatment,
+ PRIORITY_ARP_REPLY_RULE,
+ LOCAL_ENTRY_TABLE,
+ install);
+ }
+
private class InternalK8sNetworkListener implements K8sNetworkListener {
private boolean isRelevantHelper() {
@@ -189,6 +320,8 @@
}
setGatewayRule(event.subject(), true);
+ setLocalBridgeRules(event.subject(), true);
+ setLocalBridgeArpRules(event.subject(), true);
}
private void processNetworkRemoval(K8sNetworkEvent event) {
@@ -197,6 +330,8 @@
}
setGatewayRule(event.subject(), false);
+ setLocalBridgeRules(event.subject(), false);
+ setLocalBridgeArpRules(event.subject(), false);
}
}
@@ -225,6 +360,8 @@
}
k8sNetworkService.networks().forEach(n -> setGatewayRule(n, true));
+ k8sNetworkService.networks().forEach(n -> setLocalBridgeRules(n, true));
+ k8sNetworkService.networks().forEach(n -> setLocalBridgeArpRules(n, true));
}
}
}