Support internal to external communication for k8s POD using SNAT
Change-Id: I8da79d2728fc40b886e44ba4f5ea81d248e33fc2
diff --git a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
index fea0736..419e7eb 100644
--- a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
+++ b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
@@ -69,6 +69,8 @@
public static final int PRIORITY_GATEWAY_RULE = 30000;
public static final int PRIORITY_SWITCHING_RULE = 30000;
public static final int PRIORITY_CIDR_RULE = 30000;
+ public static final int PRIORITY_STATEFUL_SNAT_RULE = 41000;
+ public static final int PRIORITY_EXTERNAL_ROUTING_RULE = 25000;
public static final int PRIORITY_ARP_GATEWAY_RULE = 41000;
public static final int PRIORITY_ARP_SUBNET_RULE = 40000;
public static final int PRIORITY_ARP_CONTROL_RULE = 40000;
@@ -98,4 +100,5 @@
public static final int VTAP_OUTBOUND_MIRROR_TABLE = 72;
public static final int FORWARDING_TABLE = 80;
public static final int ERROR_TABLE = 100;
+ public static final int GW_COMMON_TABLE = 0;
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
new file mode 100644
index 0000000..349043f
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
@@ -0,0 +1,369 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import org.onlab.packet.ARP;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.TpPort;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snetworking.api.K8sFlowRuleService;
+import org.onosproject.k8snetworking.api.K8sNetwork;
+import org.onosproject.k8snetworking.api.K8sNetworkEvent;
+import org.onosproject.k8snetworking.api.K8sNetworkListener;
+import org.onosproject.k8snetworking.api.K8sNetworkService;
+import org.onosproject.k8snetworking.api.K8sPort;
+import org.onosproject.k8snetworking.util.RulePopulatorUtil;
+import org.onosproject.k8snode.api.K8sNode;
+import org.onosproject.k8snode.api.K8sNodeEvent;
+import org.onosproject.k8snode.api.K8sNodeListener;
+import org.onosproject.k8snode.api.K8sNodeService;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.DEFAULT_GATEWAY_MAC;
+import static org.onosproject.k8snetworking.api.Constants.GW_COMMON_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
+import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Provides POD's internal to external connectivity using source NAT (SNAT).
+ */
+@Component(immediate = true)
+public class K8sRoutingSnatHandler {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final int POD_PREFIX = 32;
+
+ private static final int TP_PORT_MINIMUM_NUM = 1025;
+ private static final int TP_PORT_MAXIMUM_NUM = 65535;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DriverService driverService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sNetworkService k8sNetworkService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sNodeService k8sNodeService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sFlowRuleService k8sFlowRuleService;
+
+ private final InternalK8sNetworkListener k8sNetworkListener =
+ new InternalK8sNetworkListener();
+ private final InternalK8sNodeListener k8sNodeListener =
+ new InternalK8sNodeListener();
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
+
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ k8sNetworkService.addListener(k8sNetworkListener);
+ k8sNodeService.addListener(k8sNodeListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ k8sNodeService.removeListener(k8sNodeListener);
+ k8sNetworkService.removeListener(k8sNetworkListener);
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private void setContainerToExtRule(K8sNode k8sNode, boolean install) {
+
+ K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());
+
+ if (net == null) {
+ return;
+ }
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(Long.valueOf(net.segmentId()))
+ .matchEthDst(DEFAULT_GATEWAY_MAC);
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(k8sNode.intgToExtPatchPortNum());
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_EXTERNAL_ROUTING_RULE,
+ ROUTING_TABLE,
+ install);
+ }
+
+ private void setExtToContainerRule(K8sNode k8sNode,
+ K8sPort k8sPort, boolean install) {
+
+ K8sNetwork net = k8sNetworkService.network(k8sPort.networkId());
+
+ if (net == null) {
+ return;
+ }
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sPort.ipAddress(), POD_PREFIX));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(k8sNode.extToIntgPatchPortNum());
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.extBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_STATEFUL_SNAT_RULE,
+ GW_COMMON_TABLE,
+ install);
+ }
+
+ private void setSnatDownstreamRule(K8sNode k8sNode,
+ boolean install) {
+ DeviceId deviceId = k8sNode.extBridge();
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), POD_PREFIX));
+
+ ExtensionTreatment natTreatment = RulePopulatorUtil
+ .niciraConnTrackTreatmentBuilder(driverService, deviceId)
+ .commit(false)
+ .natAction(true)
+ .table((short) 0)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setEthSrc(DEFAULT_GATEWAY_MAC)
+ .extension(natTreatment, deviceId)
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ deviceId,
+ sBuilder.build(),
+ treatment,
+ PRIORITY_STATEFUL_SNAT_RULE,
+ GW_COMMON_TABLE,
+ install);
+ }
+
+ private void setSnatUpstreamRule(K8sNode k8sNode,
+ boolean install) {
+
+ K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());
+
+ if (net == null) {
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthDst(DEFAULT_GATEWAY_MAC)
+ .build();
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ if (install) {
+ ExtensionTreatment natTreatment = RulePopulatorUtil
+ .niciraConnTrackTreatmentBuilder(driverService, k8sNode.extBridge())
+ .commit(true)
+ .natFlag(CT_NAT_SRC_FLAG)
+ .natAction(true)
+ .natIp(k8sNode.extBridgeIp())
+ .natPortMin(TpPort.tpPort(TP_PORT_MINIMUM_NUM))
+ .natPortMax(TpPort.tpPort(TP_PORT_MAXIMUM_NUM))
+ .build();
+
+ tBuilder.extension(natTreatment, k8sNode.extBridge())
+ .setEthSrc(k8sNode.extBridgeMac())
+ .setEthDst(k8sNode.extGatewayMac())
+ .setOutput(k8sNode.extBridgePortNum());
+ }
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.extBridge(),
+ selector,
+ tBuilder.build(),
+ PRIORITY_STATEFUL_SNAT_RULE,
+ GW_COMMON_TABLE,
+ install);
+ }
+
+ private void setExtIntfArpRule(K8sNode k8sNode, boolean install) {
+
+ Device device = deviceService.getDevice(k8sNode.extBridge());
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_ARP)
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(Ip4Address.valueOf(k8sNode.extBridgeIp().toString()))
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setArpOp(ARP.OP_REPLY)
+ .extension(buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(buildMoveArpShaToThaExtension(device), device.id())
+ .extension(buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpSpa(Ip4Address.valueOf(k8sNode.extBridgeIp().toString()))
+ .setArpSha(k8sNode.extBridgeMac())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.extBridge(),
+ selector,
+ treatment,
+ PRIORITY_STATEFUL_SNAT_RULE,
+ GW_COMMON_TABLE,
+ install);
+ }
+
+ private class InternalK8sNodeListener implements K8sNodeListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sNodeEvent event) {
+ switch (event.type()) {
+ case K8S_NODE_COMPLETE:
+ eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+ break;
+ case K8S_NODE_INCOMPLETE:
+ default:
+ break;
+ }
+ }
+
+ private void processNodeCompletion(K8sNode k8sNode) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setExtIntfArpRule(k8sNode, true);
+ setSnatUpstreamRule(k8sNode, true);
+ setSnatDownstreamRule(k8sNode, true);
+ setContainerToExtRule(k8sNode, true);
+ }
+ }
+
+ private class InternalK8sNetworkListener implements K8sNetworkListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sNetworkEvent event) {
+ switch (event.type()) {
+ case K8S_PORT_ACTIVATED:
+ eventExecutor.execute(() -> processPortActivation(event.port()));
+ break;
+ case K8S_PORT_REMOVED:
+ eventExecutor.execute(() -> processPortRemoval(event.port()));
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void processPortActivation(K8sPort port) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ k8sNodeService.completeNodes().forEach(n ->
+ setExtToContainerRule(n, port, true));
+ }
+
+ private void processPortRemoval(K8sPort port) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ k8sNodeService.completeNodes().forEach(n ->
+ setExtToContainerRule(n, port, false));
+ }
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
index 799ecb8..9c243f9 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
@@ -630,7 +630,9 @@
ports.forEach(p -> {
ExtensionTreatment ctNatTreatment = connTreatmentBuilder
- .natPort(TpPort.tpPort(p.getPort())).build();
+ .natPortMin(TpPort.tpPort(p.getPort()))
+ .natPortMax(TpPort.tpPort(p.getPort()))
+ .build();
ExtensionTreatment resubmitTreatment = buildResubmitExtension(
deviceService.getDevice(deviceId), ROUTING_TABLE);
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
index 89680b0..79a2530 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingArpHandler.java
@@ -252,6 +252,10 @@
}
if (replyMac == null) {
+ replyMac = MacAddress.valueOf(gatewayMac);
+ }
+
+ if (replyMac == null) {
log.debug("Failed to find MAC address for {}", targetIp);
return;
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 235897d..5ddc872 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -136,7 +136,7 @@
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
if (node.hostname().equals(k8sNetwork.name())) {
- tBuilder.setEthDst(node.intBridgeMac())
+ tBuilder.setEthDst(node.intgBridgeMac())
.setOutput(PortNumber.LOCAL);
} else {
PortNumber portNum = tunnelPortNumByNetId(k8sNetwork.networkId(),
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
index 88cb589..04fd07c 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
@@ -20,7 +20,9 @@
import org.onlab.packet.IpPrefix;
import org.onosproject.cfg.ComponentConfigService;
import org.onosproject.cfg.ConfigProperty;
+import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.k8snetworking.api.K8sFlowRuleService;
@@ -30,6 +32,8 @@
import org.onosproject.k8snetworking.api.K8sNetworkService;
import org.onosproject.k8snetworking.api.K8sPort;
import org.onosproject.k8snode.api.K8sNode;
+import org.onosproject.k8snode.api.K8sNodeEvent;
+import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
import org.onosproject.mastership.MastershipService;
import org.onosproject.net.PortNumber;
@@ -46,6 +50,7 @@
import org.osgi.service.component.annotations.ReferenceCardinality;
import org.slf4j.Logger;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutorService;
@@ -54,6 +59,7 @@
import static org.onosproject.k8snetworking.api.Constants.ACL_EGRESS_TABLE;
import static org.onosproject.k8snetworking.api.Constants.ARP_BROADCAST_MODE;
import static org.onosproject.k8snetworking.api.Constants.ARP_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.DEFAULT_GATEWAY_MAC;
import static org.onosproject.k8snetworking.api.Constants.FORWARDING_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_SWITCHING_RULE;
@@ -85,6 +91,9 @@
protected MastershipService mastershipService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
protected DeviceService deviceService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
@@ -109,13 +118,19 @@
groupedThreads(this.getClass().getSimpleName(), "event-handler"));
private final InternalK8sNetworkListener k8sNetworkListener =
new InternalK8sNetworkListener();
+ private final InternalK8sNodeListener k8sNodeListener =
+ new InternalK8sNodeListener();
private ApplicationId appId;
+ private NodeId localNodeId;
@Activate
protected void activate() {
appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
k8sNetworkService.addListener(k8sNetworkListener);
+ localNodeId = clusterService.getLocalNode().id();
+ k8sNodeService.addListener(k8sNodeListener);
+ leadershipService.runForLeadership(appId.name());
setGatewayRulesForTunnel(true);
@@ -124,6 +139,8 @@
@Deactivate
protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+ k8sNodeService.removeListener(k8sNodeListener);
k8sNetworkService.removeListener(k8sNetworkListener);
eventExecutor.shutdown();
@@ -245,6 +262,51 @@
install);
}
+ private void setExtToIntgTunnelTagFlowRules(K8sNode k8sNode, boolean install) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthSrc(DEFAULT_GATEWAY_MAC)
+ .matchInPort(k8sNode.intgToExtPatchPortNum())
+ .build();
+
+ K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.valueOf(net.segmentId()))
+ .transition(ACL_EGRESS_TABLE);
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.intgBridge(),
+ selector,
+ tBuilder.build(),
+ PRIORITY_TUNNEL_TAG_RULE,
+ VTAG_TABLE,
+ install);
+ }
+
+ private void setLocalTunnelTagFlowRules(K8sNode k8sNode, boolean install) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchInPort(PortNumber.LOCAL)
+ .build();
+
+ K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.valueOf(net.segmentId()))
+ .transition(ACL_EGRESS_TABLE);
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.intgBridge(),
+ selector,
+ tBuilder.build(),
+ PRIORITY_TUNNEL_TAG_RULE,
+ VTAG_TABLE,
+ install);
+ }
+
private void setGatewayRulesForTunnel(boolean install) {
k8sNetworkService.networks().forEach(n -> {
// switching rules for the instPorts in the same node
@@ -355,4 +417,31 @@
setNetworkRules(event.port(), false);
}
}
+
+ private class InternalK8sNodeListener implements K8sNodeListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sNodeEvent event) {
+ switch (event.type()) {
+ case K8S_NODE_COMPLETE:
+ eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void processNodeCompletion(K8sNode k8sNode) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setExtToIntgTunnelTagFlowRules(k8sNode, true);
+ setLocalTunnelTagFlowRules(k8sNode, true);
+ }
+ }
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/RulePopulatorUtil.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/RulePopulatorUtil.java
index 25a44ea..a50d30c 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/RulePopulatorUtil.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/RulePopulatorUtil.java
@@ -42,6 +42,9 @@
import static org.onosproject.k8snetworking.api.Constants.DST;
import static org.onosproject.k8snetworking.api.Constants.SRC;
import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_LOAD;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_ARP_SHA_TO_THA;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_ARP_SPA_TO_TPA;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_ETH_SRC_TO_DST;
import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_RESUBMIT_TABLE;
import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST;
import static org.slf4j.LoggerFactory.getLogger;
@@ -215,8 +218,8 @@
DeviceId deviceId,
Ip4Address remoteIp) {
Device device = deviceService.getDevice(deviceId);
- if (device != null && !device.is(ExtensionTreatmentResolver.class)) {
- log.error("The extension treatment is not supported");
+
+ if (!checkTreatmentResolver(device)) {
return null;
}
@@ -271,8 +274,7 @@
* @return resubmit extension treatment
*/
public static ExtensionTreatment buildResubmitExtension(Device device, int tableId) {
- if (device == null || !device.is(ExtensionTreatmentResolver.class)) {
- log.warn("Nicira extension treatment is not supported");
+ if (!checkTreatmentResolver(device)) {
return null;
}
@@ -301,8 +303,7 @@
public static ExtensionTreatment buildLoadExtension(Device device,
String ipType,
String shift) {
- if (device == null || !device.is(ExtensionTreatmentResolver.class)) {
- log.warn("Nicira extension treatment is not supported");
+ if (!checkTreatmentResolver(device)) {
return null;
}
@@ -335,6 +336,52 @@
}
}
+
+ /**
+ * Returns the nicira move source MAC to destination MAC extension treatment.
+ *
+ * @param device device instance
+ * @return move extension treatment
+ */
+ public static ExtensionTreatment buildMoveEthSrcToDstExtension(Device device) {
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ return resolver.getExtensionInstruction(NICIRA_MOV_ETH_SRC_TO_DST.type());
+ }
+
+ /**
+ * Returns the nicira move ARP SHA to THA extension treatment.
+ *
+ * @param device device instance
+ * @return move extension treatment
+ */
+ public static ExtensionTreatment buildMoveArpShaToThaExtension(Device device) {
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ return resolver.getExtensionInstruction(NICIRA_MOV_ARP_SHA_TO_THA.type());
+ }
+
+ /**
+ * Returns the nicira move ARP SPA to TPA extension treatment.
+ *
+ * @param device device instance
+ * @return move extension treatment
+ */
+ public static ExtensionTreatment buildMoveArpSpaToTpaExtension(Device device) {
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ return resolver.getExtensionInstruction(NICIRA_MOV_ARP_SPA_TO_TPA.type());
+ }
+
/**
* Calculate IP address upper string into integer.
*
@@ -350,6 +397,15 @@
return firstOctet << 8 | secondOctet;
}
+ private static boolean checkTreatmentResolver(Device device) {
+ if (device == null || !device.is(ExtensionTreatmentResolver.class)) {
+ log.warn("Nicira extension treatment is not supported");
+ return false;
+ }
+
+ return true;
+ }
+
/**
* Builder class for OVS Connection Tracking feature actions.
*/
@@ -358,7 +414,8 @@
private DriverService driverService;
private DeviceId deviceId;
private IpAddress natAddress = null;
- private TpPort natPort = null;
+ private TpPort natPortMin = null;
+ private TpPort natPortMax = null;
private int zone;
private boolean commit;
private short table = -1;
@@ -417,13 +474,24 @@
}
/**
- * Sets port for NAT.
+ * Sets min port for NAT.
*
* @param port port number
* @return NiciraConnTrackTreatmentBuilder object
*/
- public NiciraConnTrackTreatmentBuilder natPort(TpPort port) {
- this.natPort = port;
+ public NiciraConnTrackTreatmentBuilder natPortMin(TpPort port) {
+ this.natPortMin = port;
+ return this;
+ }
+
+ /**
+ * Sets max port for NAT.
+ *
+ * @param port port number
+ * @return NiciraConnTrackTreatmentBuilder object
+ */
+ public NiciraConnTrackTreatmentBuilder natPortMax(TpPort port) {
+ this.natPortMax = port;
return this;
}
@@ -468,19 +536,28 @@
ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_NAT.type());
try {
- natTreatment.setPropertyValue(CT_FLAGS, this.natFlag);
+ if (natAddress == null && natPortMin == null && natPortMax == null) {
+ natTreatment.setPropertyValue(CT_FLAGS, 0);
+ natTreatment.setPropertyValue(CT_PRESENT_FLAGS, 0);
+ } else {
+ natTreatment.setPropertyValue(CT_FLAGS, this.natFlag);
- natTreatment.setPropertyValue(CT_PRESENT_FLAGS,
- buildPresentFlag(natPort != null, natAddress != null));
+ natTreatment.setPropertyValue(CT_PRESENT_FLAGS,
+ buildPresentFlag((natPortMin != null && natPortMax != null),
+ natAddress != null));
+ }
if (natAddress != null) {
natTreatment.setPropertyValue(CT_IPADDRESS_MIN, natAddress);
natTreatment.setPropertyValue(CT_IPADDRESS_MAX, natAddress);
}
- if (natPort != null) {
- natTreatment.setPropertyValue(CT_PORT_MIN, natPort.toInt());
- natTreatment.setPropertyValue(CT_PORT_MAX, natPort.toInt());
+ if (natPortMin != null) {
+ natTreatment.setPropertyValue(CT_PORT_MIN, natPortMin.toInt());
+ }
+
+ if (natPortMax != null) {
+ natTreatment.setPropertyValue(CT_PORT_MAX, natPortMax.toInt());
}
} catch (Exception e) {
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
index e38d0a4..7be1e9a5 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
@@ -32,9 +32,12 @@
import java.util.Objects;
import static com.google.common.base.Preconditions.checkArgument;
+import static org.onosproject.k8snode.api.Constants.EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.GENEVE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.GRE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_EXTERNAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.PHYSICAL_EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.VXLAN_TUNNEL;
import static org.onosproject.net.AnnotationKeys.PORT_NAME;
@@ -44,7 +47,11 @@
public class DefaultK8sNode implements K8sNode {
private static final int DEFAULT_OVSDB_PORT = 6640;
+ private static final String IP_ADDRESS = "ip_address";
private static final String MAC_ADDRESS = "mac_address";
+ private static final String EXT_INTF = "ext_interface";
+ private static final String EXT_GW_IP = "ext_gw_ip_address";
+ private static final String EXT_GW_MAC = "ext_gw_mac_address";
private final String hostname;
private final Type type;
@@ -175,7 +182,7 @@
}
@Override
- public PortNumber intBridgePortNum() {
+ public PortNumber intgBridgePortNum() {
DeviceService deviceService = DefaultServiceDirectory.getService(DeviceService.class);
Port port = deviceService.getPorts(intgBridge).stream()
.filter(p -> p.isEnabled() &&
@@ -185,21 +192,99 @@
}
@Override
- public MacAddress intBridgeMac() {
- OvsdbController ovsdbController =
- DefaultServiceDirectory.getService(OvsdbController.class);
- OvsdbNodeId ovsdb = new OvsdbNodeId(this.managementIp, DEFAULT_OVSDB_PORT);
- OvsdbClientService client = ovsdbController.getOvsdbClient(ovsdb);
+ public MacAddress intgBridgeMac() {
+ OvsdbClientService client = getOvsClient();
+
if (client == null) {
return null;
}
- Interface iface = client.getInterface(INTEGRATION_BRIDGE);
+ Interface iface = getOvsClient().getInterface(INTEGRATION_BRIDGE);
OvsdbMap data = (OvsdbMap) iface.getExternalIdsColumn().data();
return MacAddress.valueOf((String) data.map().get(MAC_ADDRESS));
}
@Override
+ public IpAddress extBridgeIp() {
+ OvsdbClientService client = getOvsClient();
+
+ if (client == null) {
+ return null;
+ }
+
+ Interface iface = getOvsClient().getInterface(EXTERNAL_BRIDGE);
+ OvsdbMap data = (OvsdbMap) iface.getExternalIdsColumn().data();
+ return IpAddress.valueOf((String) data.map().get(IP_ADDRESS));
+ }
+
+ @Override
+ public MacAddress extBridgeMac() {
+ OvsdbClientService client = getOvsClient();
+
+ if (client == null) {
+ return null;
+ }
+
+ Interface iface = getOvsClient().getInterface(EXTERNAL_BRIDGE);
+ OvsdbMap data = (OvsdbMap) iface.getExternalIdsColumn().data();
+ return MacAddress.valueOf((String) data.map().get(MAC_ADDRESS));
+ }
+
+ @Override
+ public IpAddress extGatewayIp() {
+ OvsdbClientService client = getOvsClient();
+
+ if (client == null) {
+ return null;
+ }
+
+ Interface iface = getOvsClient().getInterface(EXTERNAL_BRIDGE);
+ OvsdbMap data = (OvsdbMap) iface.getExternalIdsColumn().data();
+ return IpAddress.valueOf((String) data.map().get(EXT_GW_IP));
+ }
+
+ @Override
+ public MacAddress extGatewayMac() {
+ OvsdbClientService client = getOvsClient();
+
+ if (client == null) {
+ return null;
+ }
+
+ Interface iface = getOvsClient().getInterface(EXTERNAL_BRIDGE);
+ OvsdbMap data = (OvsdbMap) iface.getExternalIdsColumn().data();
+ return MacAddress.valueOf((String) data.map().get(EXT_GW_MAC));
+ }
+
+ @Override
+ public PortNumber intgToExtPatchPortNum() {
+ return portNumber(intgBridge, INTEGRATION_TO_EXTERNAL_BRIDGE);
+ }
+
+ @Override
+ public PortNumber extToIntgPatchPortNum() {
+ return portNumber(extBridge, PHYSICAL_EXTERNAL_BRIDGE);
+ }
+
+ @Override
+ public PortNumber extBridgePortNum() {
+ OvsdbClientService client = getOvsClient();
+
+ if (client == null) {
+ return null;
+ }
+
+ Interface iface = getOvsClient().getInterface(EXTERNAL_BRIDGE);
+ OvsdbMap data = (OvsdbMap) iface.getExternalIdsColumn().data();
+ String extIface = (String) data.map().get(EXT_INTF);
+ if (extIface == null) {
+ return null;
+ }
+
+ return portNumber(extBridge, extIface);
+ }
+
+ @Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
@@ -243,14 +328,31 @@
if (dataIp == null) {
return null;
}
+
+ return portNumber(intgBridge, tunnelType);
+ }
+
+ private PortNumber portNumber(DeviceId deviceId, String portName) {
DeviceService deviceService = DefaultServiceDirectory.getService(DeviceService.class);
- Port port = deviceService.getPorts(intgBridge).stream()
+ Port port = deviceService.getPorts(deviceId).stream()
.filter(p -> p.isEnabled() &&
- Objects.equals(p.annotations().value(PORT_NAME), tunnelType))
+ Objects.equals(p.annotations().value(PORT_NAME), portName))
.findAny().orElse(null);
return port != null ? port.number() : null;
}
+ private OvsdbClientService getOvsClient() {
+ OvsdbController ovsdbController =
+ DefaultServiceDirectory.getService(OvsdbController.class);
+ OvsdbNodeId ovsdb = new OvsdbNodeId(this.managementIp, DEFAULT_OVSDB_PORT);
+ OvsdbClientService client = ovsdbController.getOvsdbClient(ovsdb);
+ if (client == null) {
+ return null;
+ }
+
+ return client;
+ }
+
/**
* Returns new builder instance.
*
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
index 6dfbead..30449ff 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
@@ -146,14 +146,63 @@
*
* @return host port number; null if the host port does not exist
*/
- PortNumber intBridgePortNum();
+ PortNumber intgBridgePortNum();
/**
* Returns the integration bridge's MAC address.
*
* @return MAC address; null if the MAC address does not exist
*/
- MacAddress intBridgeMac();
+ MacAddress intgBridgeMac();
+
+ /**
+ * Returns the external bridge's IP address.
+ *
+ * @return IP address; null if the IP address does not exist
+ */
+ IpAddress extBridgeIp();
+
+ /**
+ * Returns the external bridge's MAC address.
+ *
+ * @return MAC address; null if the MAC address does not exist
+ */
+ MacAddress extBridgeMac();
+
+ /**
+ * Returns the external gateway IP address.
+ *
+ * @return IP address; null if the IP address does not exist
+ */
+ IpAddress extGatewayIp();
+
+ /**
+ * Returns the external gateway MAC address.
+ *
+ * @return MAC address; null if the MAC address does not exist
+ */
+ MacAddress extGatewayMac();
+
+ /**
+ * Returns the integration to external patch port number.
+ *
+ * @return patch port number
+ */
+ PortNumber intgToExtPatchPortNum();
+
+ /**
+ * Returns the external to integration patch port number.
+ *
+ * @return patch port number
+ */
+ PortNumber extToIntgPatchPortNum();
+
+ /**
+ * Returns the external bridge to router port number.
+ *
+ * @return port number, null if the port does not exist
+ */
+ PortNumber extBridgePortNum();
/**
* Builder of new node entity.