Refactor channel and mastership handling in P4Runtime
This (big) change aims at solving the issue observed with mastership flapping
and device connection/disconnection with P4Runtime.
Channel handling is now based on the underlying gRPC channel state. Before,
channel events (open/close/error) were generated as a consequence of P4Runtime
StreamChannel events, making device availability dependent on mastership. Now
Stream Channel events only affect mastership (MASTER/STANDBY or NONE when the
SteamChannel RPC is not active).
Mastership handling has been refactored to generate P4Runtime election IDs that
are compatible with the mastership preference decided by the MastershipService.
GeneralDeviceProvider has been re-implemented to support in-order
device event processing and to reduce implementation complexity. Stats polling
has been moved to a separate component, and netcfg handling updated to only
depend on BasicDeviceConfig, augmented with a pipeconf field, and re-using the
managementAddress field to set the gRPC server endpoints (e.g.
grpc://myswitch.local:50051). Before it was depending on 3 different config
classes, making hard to detect changes.
Finally, this change affects some core interfaces:
- Adds a method to DeviceProvider and DeviceHandshaker to check for device
availability, making the meaning of availability device-specific. This is needed
in cases where the device manager needs to change the availability state of a
device (as in change #20842)
- Support device providers not capable of reconciling mastership role responses
with requests (like P4Runtime).
- Clarify the meaning of "connection" in the DeviceConnect behavior.
- Allows driver-based providers to check devices for reachability and
availability without probing the device via the network.
Change-Id: I7ff30d29f5d02ad938e3171536e54ae2916629a2
diff --git a/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcChannelController.java b/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcChannelController.java
index 5b07254..ad4324b 100644
--- a/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcChannelController.java
+++ b/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcChannelController.java
@@ -23,6 +23,7 @@
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
/**
* Abstraction of a gRPC controller that stores and manages gRPC channels.
@@ -33,10 +34,10 @@
int CONNECTION_TIMEOUT_SECONDS = 20;
/**
- * Creates a gRPC managed channel from the given builder and opens a
- * connection to it. If the connection is successful returns the managed
- * channel object and stores the channel internally, associated with the
- * given channel ID.
+ * Creates a gRPC managed channel from the given builder and opens the
+ * connection. If the connection is successful, returns the managed channel
+ * object and stores the channel internally, associated with the given
+ * channel ID.
* <p>
* This method blocks until the channel is open or a timeout expires. By
* default the timeout is {@link #CONNECTION_TIMEOUT_SECONDS} seconds. If
@@ -72,15 +73,6 @@
Map<GrpcChannelId, ManagedChannel> getChannels();
/**
- * Returns true if the channel associated with the given identifier is open,
- * i.e. the server is able to successfully reply to RPCs, false otherwise.
- *
- * @param channelId channel ID
- * @return true if channel is open, false otherwise.
- */
- boolean isChannelOpen(GrpcChannelId channelId);
-
- /**
* If present, returns the channel associated with the given ID.
*
* @param channelId channel ID
@@ -88,4 +80,14 @@
*/
Optional<ManagedChannel> getChannel(GrpcChannelId channelId);
+ /**
+ * Probes the server at the endpoint of the given channel. Returns true if
+ * the server responded to the probe, false otherwise or if the channel does
+ * not exist.
+ *
+ * @param channelId channel ID
+ * @return completable future eventually true if the gRPC server responded
+ * to the probe; false otherwise
+ */
+ CompletableFuture<Boolean> probeChannel(GrpcChannelId channelId);
}
diff --git a/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClient.java b/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClient.java
index d040a23..d529529 100644
--- a/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClient.java
+++ b/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClient.java
@@ -17,11 +17,11 @@
package org.onosproject.grpc.api;
import com.google.common.annotations.Beta;
+
import java.util.concurrent.CompletableFuture;
/**
* Abstraction of a gRPC client.
- *
*/
@Beta
public interface GrpcClient {
@@ -33,4 +33,30 @@
* procedure
*/
CompletableFuture<Void> shutdown();
+
+ /**
+ * This method provides a coarse modelling of gRPC channel {@link
+ * io.grpc.ConnectivityState}. The implementation does not make any attempt
+ * at probing the server by sending messages over the network, as such it
+ * does not block execution. It returns true if this client is expected to
+ * communicate with the server successfully. In other words, if any RPC
+ * would be executed immediately after this method is called and returns
+ * true, then it is expected, but NOT guaranteed, for that RPC message to
+ * reach the server and be processed. If false, it means the channel is in a
+ * failure state and communication with the server is unlikely to happen
+ * soon.
+ *
+ * @return true if server is deemed reachable, false otherwise
+ */
+ boolean isServerReachable();
+
+ /**
+ * Similar to {@link #isServerReachable()}, but might involve sending
+ * packets over the network. This checks whether the specific gRPC
+ * service(s) required by this client is available or not on the server.
+ *
+ * @return completable future eventually true if the gRPC service(s) on the
+ * server was available during the probe; false otherwise
+ */
+ CompletableFuture<Boolean> probeService();
}
diff --git a/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClientController.java b/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClientController.java
index 3cdbcd1..b47d499 100644
--- a/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClientController.java
+++ b/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClientController.java
@@ -18,10 +18,11 @@
import com.google.common.annotations.Beta;
import org.onosproject.net.DeviceId;
+import org.onosproject.net.device.DeviceAgentListener;
+import org.onosproject.net.provider.ProviderId;
/**
- * Abstraction of a gRPC controller which controls specific gRPC client {@link
- * C} with specific client key {@link K}.
+ * Abstraction of controller that manages gRPC clients.
*
* @param <K> the gRPC client key
* @param <C> the gRPC client type
@@ -34,20 +35,27 @@
* given information. As a result of this method, a client can be later
* obtained by invoking {@link #getClient(DeviceId)}.
* <p>
+ * Upon creation, a connection to the server is automatically started, which
+ * blocks execution. If the connection is successful, the client is created
+ * and this method returns true, otherwise (e.g., socket error) any state
+ * associated with this client is destroyed and returns false.
+ * <p>
* Only one client can exist for the same device ID. Calls to this method
* are idempotent fot the same client key, i.e. returns true if such client
- * already exists but a new one is not created. If there exists a client
- * with same device ID but different address and port, removes old one and
- * recreate new one.
+ * already exists. Otherwise, if a client for the same device ID but
+ * different client key already exists, throws an exception.
*
* @param clientKey the client key
* @return true if the client was created and the channel to the server is
* open; false otherwise
+ * @throws IllegalArgumentException if a client for the same device ID but
+ * different client key already exists.
*/
boolean createClient(K clientKey);
/**
- * Retrieves the gRPC client to operate on the given device.
+ * Returns the gRPC client previously created for the given device, or null
+ * if such client does not exist.
*
* @param deviceId the device identifier
* @return the gRPC client of the device if exists; null otherwise
@@ -55,24 +63,47 @@
C getClient(DeviceId deviceId);
/**
- * Removes the gRPC client for the given device. If no client exists for the
- * given device, the result is a no-op.
+ * Returns the gRPC client previously created for the given client key, or
+ * null if such client does not exist.
+ *
+ * @param clientKey client key
+ * @return the gRPC client of the device if exists; null otherwise
+ */
+ C getClient(K clientKey);
+
+ /**
+ * Removes the gRPC client for the given device and any gRPC channel state
+ * associated to it. If no client exists for the given device, the result is
+ * a no-op.
*
* @param deviceId the device identifier
*/
void removeClient(DeviceId deviceId);
/**
- * Check reachability of the gRPC server running on the given device.
- * Reachability can be tested only if a client is previously created using
- * {@link #createClient(GrpcClientKey)}. Note that this only checks the
- * reachability instead of checking service availability, different
- * service-specific gRPC clients might check service availability in a
- * different way.
+ * Similar to {@link #removeClient(DeviceId)} but uses the client key to
+ * identify the client to remove.
*
- * @param deviceId the device identifier
- * @return true if client was created and is able to contact the gNMI
- * server; false otherwise
+ * @param clientKey the client key
*/
- boolean isReachable(DeviceId deviceId);
+ void removeClient(K clientKey);
+
+ /**
+ * Adds a listener for device agent events for the given provider.
+ *
+ * @param deviceId device identifier
+ * @param providerId provider ID
+ * @param listener the device agent listener
+ */
+ void addDeviceAgentListener(DeviceId deviceId, ProviderId providerId,
+ DeviceAgentListener listener);
+
+ /**
+ * Removes the listener for device agent events that was previously
+ * registered for the given provider.
+ *
+ * @param deviceId device identifier
+ * @param providerId the provider ID
+ */
+ void removeDeviceAgentListener(DeviceId deviceId, ProviderId providerId);
}
diff --git a/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClientKey.java b/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClientKey.java
index 99ea23f..d1d0b0f 100644
--- a/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClientKey.java
+++ b/protocols/grpc/api/src/main/java/org/onosproject/grpc/api/GrpcClientKey.java
@@ -20,8 +20,11 @@
import com.google.common.base.Objects;
import org.onosproject.net.DeviceId;
+import java.net.URI;
+
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Strings.isNullOrEmpty;
import static java.lang.String.format;
/**
@@ -29,32 +32,36 @@
*/
@Beta
public class GrpcClientKey {
+
+ private static final String GRPC = "grpc";
+ private static final String GRPCS = "grpcs";
+
private final String serviceName;
private final DeviceId deviceId;
- private final String serverAddr;
- private final int serverPort;
+ private final URI serverUri;
/**
* Creates a new client key.
*
* @param serviceName gRPC service name of the client
- * @param deviceId ONOS device ID
- * @param serverAddr gRPC server address
- * @param serverPort gRPC server port
+ * @param deviceId ONOS device ID
+ * @param serverUri gRPC server URI
*/
- public GrpcClientKey(String serviceName, DeviceId deviceId, String serverAddr, int serverPort) {
+ public GrpcClientKey(String serviceName, DeviceId deviceId, URI serverUri) {
checkNotNull(serviceName);
checkNotNull(deviceId);
- checkNotNull(serverAddr);
+ checkNotNull(serverUri);
checkArgument(!serviceName.isEmpty(),
- "Service name can not be null");
- checkArgument(!serverAddr.isEmpty(),
- "Server address should not be empty");
- checkArgument(serverPort > 0 && serverPort <= 65535, "Invalid server port");
+ "Service name can not be null");
+ checkArgument(serverUri.getScheme().equals(GRPC)
+ || serverUri.getScheme().equals(GRPCS),
+ format("Server URI scheme must be %s or %s", GRPC, GRPCS));
+ checkArgument(!isNullOrEmpty(serverUri.getHost()),
+ "Server host address should not be empty");
+ checkArgument(serverUri.getPort() > 0 && serverUri.getPort() <= 65535, "Invalid server port");
this.serviceName = serviceName;
this.deviceId = deviceId;
- this.serverAddr = serverAddr;
- this.serverPort = serverPort;
+ this.serverUri = serverUri;
}
/**
@@ -76,21 +83,21 @@
}
/**
- * Gets the gRPC server address.
+ * Returns the gRPC server URI.
*
- * @return the gRPC server address.
+ * @return the gRPC server URI.
*/
- public String serverAddr() {
- return serverAddr;
+ public URI serveUri() {
+ return serverUri;
}
/**
- * Gets the gRPC server port.
+ * Returns true if the client requires TLS/SSL, false otherwise.
*
- * @return the gRPC server port.
+ * @return boolean
*/
- public int serverPort() {
- return serverPort;
+ public boolean requiresSecureChannel() {
+ return serverUri.getScheme().equals(GRPCS);
}
@Override
@@ -102,19 +109,18 @@
return false;
}
GrpcClientKey that = (GrpcClientKey) o;
- return serverPort == that.serverPort &&
- Objects.equal(serviceName, that.serviceName) &&
+ return Objects.equal(serviceName, that.serviceName) &&
Objects.equal(deviceId, that.deviceId) &&
- Objects.equal(serverAddr, that.serverAddr);
+ Objects.equal(serverUri, that.serverUri);
}
@Override
public int hashCode() {
- return Objects.hashCode(serviceName, deviceId, serverAddr, serverPort);
+ return Objects.hashCode(serviceName, deviceId, serverUri);
}
@Override
public String toString() {
- return format("%s/%s@%s:%s", deviceId, serviceName, serverAddr, serverPort);
+ return format("%s/%s@%s", deviceId, serviceName, serverUri);
}
}