Fix: inject correct default k8s service CIDR with minor refactoring
Change-Id: I1a402e11c8455d4d5a2a03845df9f4e48897efb7
diff --git a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
index 1c3ee7f..fea0736 100644
--- a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
+++ b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
@@ -49,9 +49,6 @@
public static final String SRC = "src";
public static final String DST = "dst";
- // TODO: need to inject service IP CIDR through REST
- public static final String SERVICE_IP_CIDR = "10.96.0.0/24";
-
public static final String PORT_NAME_PREFIX_CONTAINER = "veth";
public static final String ANNOTATION_NETWORK_ID = "networkId";
@@ -68,6 +65,7 @@
public static final int PRIORITY_CT_RULE = 32000;
public static final int PRIORITY_CT_DROP_RULE = 32500;
public static final int PRIORITY_NAT_RULE = 30000;
+ public static final int PRIORITY_SERVICE_CIDR_RULE = 32000;
public static final int PRIORITY_GATEWAY_RULE = 30000;
public static final int PRIORITY_SWITCHING_RULE = 30000;
public static final int PRIORITY_CIDR_RULE = 30000;
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
index f6f1033..4ed2f99 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
@@ -63,12 +63,8 @@
import static org.onosproject.k8snetworking.api.Constants.JUMP_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_CIDR_RULE;
-import static org.onosproject.k8snetworking.api.Constants.PRIORITY_CT_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_SNAT_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
-import static org.onosproject.k8snetworking.api.Constants.SERVICE_FAKE_MAC_STR;
-import static org.onosproject.k8snetworking.api.Constants.SERVICE_IP_CIDR;
-import static org.onosproject.k8snetworking.api.Constants.SHIFTED_IP_CIDR;
import static org.onosproject.k8snetworking.api.Constants.STAT_INBOUND_TABLE;
import static org.onosproject.k8snetworking.api.Constants.STAT_OUTBOUND_TABLE;
import static org.onosproject.k8snetworking.api.Constants.VTAG_TABLE;
@@ -336,38 +332,6 @@
}
}
- private void setGroupingRule(IpPrefix srcPrefix) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPSrc(srcPrefix);
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .transition(ROUTING_TABLE);
-
- for (K8sNode node : k8sNodeService.completeNodes()) {
- FlowRule flowRule = DefaultFlowRule.builder()
- .forDevice(node.intgBridge())
- .withSelector(sBuilder.build())
- .withTreatment(tBuilder.build())
- .withPriority(PRIORITY_CT_RULE)
- .fromApp(appId)
- .makePermanent()
- .forTable(JUMP_TABLE)
- .build();
- applyRule(flowRule, true);
- }
- }
-
- private void setupTransientRoutingRule() {
- setGroupingRule(IpPrefix.valueOf(SHIFTED_IP_CIDR));
- }
-
- private void setupServiceRoutingRule(K8sNetwork k8sNetwork) {
- setGroupingRule(IpPrefix.valueOf(SERVICE_IP_CIDR));
- setAnyRoutingRule(IpPrefix.valueOf(SERVICE_IP_CIDR),
- MacAddress.valueOf(SERVICE_FAKE_MAC_STR), k8sNetwork);
- }
-
private void setupHostRoutingRule(K8sNetwork k8sNetwork) {
setAnyRoutingRule(IpPrefix.valueOf(
k8sNetwork.gatewayIp(), 32), null, k8sNetwork);
@@ -399,12 +363,8 @@
}
initializePipeline(node);
- setupTransientRoutingRule();
- k8sNetworkService.networks().forEach(n -> {
- setupHostRoutingRule(n);
- setupServiceRoutingRule(n);
- });
+ k8sNetworkService.networks().forEach(K8sFlowRuleManager.this::setupHostRoutingRule);
}
}
@@ -434,7 +394,6 @@
}
setupHostRoutingRule(network);
- setupServiceRoutingRule(network);
}
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
index 26d8400..76785df 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
@@ -28,6 +28,7 @@
import org.onlab.packet.IPv4;
import org.onlab.packet.IpAddress;
import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
import org.onlab.packet.TpPort;
import org.onlab.util.Tools;
import org.onosproject.cfg.ComponentConfigService;
@@ -42,6 +43,9 @@
import org.onosproject.k8snetworking.api.K8sEndpointsService;
import org.onosproject.k8snetworking.api.K8sFlowRuleService;
import org.onosproject.k8snetworking.api.K8sGroupRuleService;
+import org.onosproject.k8snetworking.api.K8sNetwork;
+import org.onosproject.k8snetworking.api.K8sNetworkEvent;
+import org.onosproject.k8snetworking.api.K8sNetworkListener;
import org.onosproject.k8snetworking.api.K8sNetworkService;
import org.onosproject.k8snetworking.api.K8sPodEvent;
import org.onosproject.k8snetworking.api.K8sPodListener;
@@ -56,6 +60,7 @@
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.DriverService;
import org.onosproject.net.flow.DefaultTrafficSelector;
@@ -93,20 +98,25 @@
import static org.onosproject.k8snetworking.api.Constants.NAT_STATELESS;
import static org.onosproject.k8snetworking.api.Constants.NAT_TABLE;
import static org.onosproject.k8snetworking.api.Constants.POD_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_CIDR_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_CT_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_INTER_ROUTING_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_NAT_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
-import static org.onosproject.k8snetworking.api.Constants.SERVICE_IP_CIDR;
+import static org.onosproject.k8snetworking.api.Constants.SERVICE_FAKE_MAC_STR;
import static org.onosproject.k8snetworking.api.Constants.SERVICE_TABLE;
import static org.onosproject.k8snetworking.api.Constants.SHIFTED_IP_CIDR;
import static org.onosproject.k8snetworking.api.Constants.SHIFTED_IP_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.SRC;
+import static org.onosproject.k8snetworking.api.Constants.STAT_OUTBOUND_TABLE;
+import static org.onosproject.k8snetworking.impl.OsgiPropertyConstants.SERVICE_CIDR;
+import static org.onosproject.k8snetworking.impl.OsgiPropertyConstants.SERVICE_IP_CIDR_DEFAULT;
import static org.onosproject.k8snetworking.impl.OsgiPropertyConstants.SERVICE_IP_NAT_MODE;
import static org.onosproject.k8snetworking.impl.OsgiPropertyConstants.SERVICE_IP_NAT_MODE_DEFAULT;
-import static org.onosproject.k8snetworking.api.Constants.STAT_OUTBOUND_TABLE;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.nodeIpGatewayIpMap;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.tunnelPortNumByNetId;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.CT_NAT_DST_FLAG;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildGroupBucket;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildLoadExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildResubmitExtension;
@@ -122,7 +132,8 @@
@Component(
immediate = true,
property = {
- SERVICE_IP_NAT_MODE + "=" + SERVICE_IP_NAT_MODE_DEFAULT
+ SERVICE_IP_NAT_MODE + "=" + SERVICE_IP_NAT_MODE_DEFAULT,
+ SERVICE_CIDR + "=" + SERVICE_IP_CIDR_DEFAULT
}
)
public class K8sServiceHandler {
@@ -135,6 +146,7 @@
private static final String CLUSTER_IP = "ClusterIP";
private static final String TCP = "TCP";
private static final String UDP = "UDP";
+ private static final String SERVICE_IP_NAT_MODE = "serviceIpNatMode";
private static final String GROUP_ID_COUNTER_NAME = "group-id-counter";
@@ -186,6 +198,9 @@
/** Service IP address translation mode. */
private String serviceIpNatMode = SERVICE_IP_NAT_MODE_DEFAULT;
+ /** Ranges of IP address of service VIP. */
+ private String serviceCidr = SERVICE_IP_CIDR_DEFAULT;
+
private final ExecutorService eventExecutor = newSingleThreadExecutor(
groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
private final InternalNodeEventListener internalNodeEventListener =
@@ -196,6 +211,8 @@
new InternalK8sPodListener();
private final InternalK8sEndpointsListener internalK8sEndpointsListener =
new InternalK8sEndpointsListener();
+ private final InternalK8sNetworkListener internalK8sNetworkListener =
+ new InternalK8sNetworkListener();
private AtomicCounter groupIdCounter;
@@ -212,6 +229,7 @@
k8sServiceService.addListener(internalK8sServiceListener);
k8sPodService.addListener(internalK8sPodListener);
k8sEndpointsService.addListener(internalK8sEndpointsListener);
+ k8sNetworkService.addListener(internalK8sNetworkListener);
groupIdCounter = storageService.getAtomicCounter(GROUP_ID_COUNTER_NAME);
@@ -225,6 +243,7 @@
k8sNodeService.removeListener(internalNodeEventListener);
k8sServiceService.removeListener(internalK8sServiceListener);
k8sEndpointsService.removeListener(internalK8sEndpointsListener);
+ k8sNetworkService.removeListener(internalK8sNetworkListener);
configService.unregisterProperties(getClass(), false);
eventExecutor.shutdown();
@@ -245,7 +264,7 @@
k8sNetworkService.networks().forEach(n -> {
// TODO: need to provide a way to add multiple service IP CIDR ranges
- setUntrack(deviceId, ctUntrack, ctMaskUntrack, n.cidr(), SERVICE_IP_CIDR,
+ setUntrack(deviceId, ctUntrack, ctMaskUntrack, n.cidr(), serviceCidr,
JUMP_TABLE, NAT_TABLE, PRIORITY_CT_RULE, install);
setUntrack(deviceId, ctUntrack, ctMaskUntrack, n.cidr(), n.cidr(),
JUMP_TABLE, ROUTING_TABLE, PRIORITY_CT_RULE, install);
@@ -274,7 +293,7 @@
k8sNodeService.node(deviceId).hostname()).cidr();
k8sNetworkService.networks().forEach(n -> {
- setSrcDstCidrRules(deviceId, n.cidr(), SERVICE_IP_CIDR, null, JUMP_TABLE,
+ setSrcDstCidrRules(deviceId, n.cidr(), serviceCidr, null, JUMP_TABLE,
SERVICE_TABLE, PRIORITY_CT_RULE, install);
setSrcDstCidrRules(deviceId, n.cidr(), SHIFTED_IP_CIDR, null, JUMP_TABLE,
POD_TABLE, PRIORITY_CT_RULE, install);
@@ -314,10 +333,6 @@
install);
}
- private String servicePortStr(String ip, int port, String protocol) {
- return ip + "_" + port + "_" + protocol;
- }
-
/**
* Obtains the service port to endpoint address paired map.
*
@@ -558,6 +573,52 @@
install);
}
+ private void setCidrRoutingRule(IpPrefix prefix, MacAddress mac,
+ K8sNetwork network, boolean install) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(prefix)
+ .matchIPDst(IpPrefix.valueOf(network.cidr()));
+
+ k8sNodeService.completeNodes().forEach(n -> {
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.valueOf(network.segmentId()));
+
+ if (n.hostname().equals(network.name())) {
+ if (mac != null) {
+ tBuilder.setEthSrc(mac);
+ }
+ tBuilder.transition(STAT_OUTBOUND_TABLE);
+ } else {
+ PortNumber portNum = tunnelPortNumByNetId(network.networkId(),
+ k8sNetworkService, n);
+ K8sNode localNode = k8sNodeService.node(network.name());
+
+ tBuilder.extension(buildExtension(
+ deviceService,
+ n.intgBridge(),
+ localNode.dataIp().getIp4Address()),
+ n.intgBridge())
+ .setOutput(portNum);
+ }
+
+ k8sFlowRuleService.setRule(
+ appId,
+ n.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_CIDR_RULE,
+ ROUTING_TABLE,
+ install
+ );
+ });
+ }
+
+ private void setupServiceDefaultRule(K8sNetwork k8sNetwork, boolean install) {
+ setCidrRoutingRule(IpPrefix.valueOf(serviceCidr),
+ MacAddress.valueOf(SERVICE_FAKE_MAC_STR), k8sNetwork, install);
+ }
+
private void setStatefulGroupFlowRules(DeviceId deviceId, long ctState,
long ctMask, Service service,
boolean install) {
@@ -727,6 +788,10 @@
}
}
+ private String servicePortStr(String ip, int port, String protocol) {
+ return ip + "_" + port + "_" + protocol;
+ }
+
/**
* Extracts properties from the component configuration context.
*
@@ -738,6 +803,11 @@
String updatedNatMode = Tools.get(properties, SERVICE_IP_NAT_MODE);
serviceIpNatMode = updatedNatMode != null ? updatedNatMode : SERVICE_IP_NAT_MODE_DEFAULT;
log.info("Configured. Service IP NAT mode is {}", serviceIpNatMode);
+
+ String updatedServiceCidr = Tools.get(properties, SERVICE_CIDR);
+ serviceCidr = updatedServiceCidr != null ?
+ updatedServiceCidr : SERVICE_IP_CIDR_DEFAULT;
+ log.info("Configured. Service VIP range is {}", serviceCidr);
}
private void setServiceNatRules(DeviceId deviceId, boolean install) {
@@ -926,8 +996,36 @@
}
setServiceNatRules(node.intgBridge(), true);
-
k8sEndpointsService.endpointses().forEach(e -> setK8sApiRules(node, e, true));
+ k8sNetworkService.networks().forEach(n -> setupServiceDefaultRule(n, true));
+ }
+ }
+
+ private class InternalK8sNetworkListener implements K8sNetworkListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sNetworkEvent event) {
+ switch (event.type()) {
+ case K8S_NETWORK_CREATED:
+ eventExecutor.execute(() -> processNetworkCreation(event.subject()));
+ break;
+ case K8S_NETWORK_UPDATED:
+ case K8S_NETWORK_REMOVED:
+ default:
+ break;
+ }
+ }
+
+ private void processNetworkCreation(K8sNetwork network) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setupServiceDefaultRule(network, true);
}
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/OsgiPropertyConstants.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/OsgiPropertyConstants.java
index 04705f3..6a6979b 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/OsgiPropertyConstants.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/OsgiPropertyConstants.java
@@ -34,4 +34,6 @@
static final String SERVICE_IP_NAT_MODE = "serviceIpNatMode";
static final String SERVICE_IP_NAT_MODE_DEFAULT = "stateless";
+ static final String SERVICE_CIDR = "serviceCidr";
+ static final String SERVICE_IP_CIDR_DEFAULT = "10.96.0.0/12";
}