Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2017-present Open Networking Foundation |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package org.onosproject.drivers.barefoot; |
| 18 | |
| 19 | import com.google.common.collect.ImmutableList; |
| 20 | import com.google.common.collect.Lists; |
| 21 | import org.apache.commons.io.IOUtils; |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 22 | import org.apache.thrift.TException; |
| 23 | import org.apache.thrift.protocol.TBinaryProtocol; |
| 24 | import org.apache.thrift.protocol.TMultiplexedProtocol; |
| 25 | import org.apache.thrift.protocol.TProtocol; |
| 26 | import org.apache.thrift.transport.TSocket; |
| 27 | import org.apache.thrift.transport.TTransport; |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 28 | import org.onlab.util.SharedExecutors; |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 29 | import org.onosproject.drivers.barefoot.pal.pal; |
Carmelo Cascone | 254683f | 2017-12-18 17:56:27 -0800 | [diff] [blame^] | 30 | import org.onosproject.drivers.barefoot.pal.pal_autoneg_policy_t; |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 31 | import org.onosproject.drivers.barefoot.pal.pal_fec_type_t; |
| 32 | import org.onosproject.drivers.barefoot.pal.pal_port_speed_t; |
| 33 | import org.onosproject.incubator.net.config.basics.PortDescriptionsConfig; |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 34 | import org.onosproject.net.DeviceId; |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 35 | import org.onosproject.net.config.NetworkConfigService; |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 36 | import org.onosproject.net.device.PortDescription; |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 37 | import org.onosproject.net.driver.AbstractHandlerBehaviour; |
| 38 | import org.onosproject.net.pi.model.PiPipeconf; |
| 39 | import org.onosproject.net.pi.model.PiPipeconf.ExtensionType; |
| 40 | import org.onosproject.net.pi.model.PiPipelineProgrammable; |
| 41 | import org.onosproject.p4runtime.api.P4RuntimeClient; |
| 42 | import org.onosproject.p4runtime.api.P4RuntimeController; |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 43 | import org.onosproject.provider.general.device.api.GeneralProviderDeviceConfig; |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 44 | import org.slf4j.Logger; |
| 45 | |
| 46 | import java.io.IOException; |
| 47 | import java.io.InputStream; |
| 48 | import java.nio.Buffer; |
| 49 | import java.nio.ByteBuffer; |
| 50 | import java.nio.ByteOrder; |
| 51 | import java.nio.charset.StandardCharsets; |
| 52 | import java.util.List; |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 53 | import java.util.Map; |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 54 | import java.util.Optional; |
| 55 | import java.util.concurrent.CompletableFuture; |
| 56 | import java.util.concurrent.ExecutionException; |
| 57 | |
| 58 | import static com.google.common.base.Strings.isNullOrEmpty; |
| 59 | import static org.slf4j.LoggerFactory.getLogger; |
| 60 | |
| 61 | /** |
| 62 | * Implementation of the PiPipelineProgrammable for BMv2. |
| 63 | */ |
| 64 | public class TofinoPipelineProgrammable extends AbstractHandlerBehaviour implements PiPipelineProgrammable { |
| 65 | |
Carmelo Cascone | 5db3968 | 2017-09-07 16:36:42 +0200 | [diff] [blame] | 66 | // FIXME: default pipeconf should depend on the system. Maybe pass it via netcfg? |
| 67 | private static final PiPipeconf DEFAULT_PIPECONF = TofinoDefaultPipeconfFactory.getMavericks(); |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 68 | private static final String P4RUNTIME = "p4runtime"; |
| 69 | private static final String PROTOCOL_KEY_IP = "ip"; |
| 70 | private static final String DEVICE_ID = "deviceId"; |
| 71 | private static final int PAL_MGMT_PORT = 9090; |
| 72 | private static final String PAL_THRIFT_SERVICE = "pal"; |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 73 | |
| 74 | private final Logger log = getLogger(getClass()); |
| 75 | |
| 76 | @Override |
| 77 | public CompletableFuture<Boolean> deployPipeconf(PiPipeconf pipeconf) { |
| 78 | |
| 79 | CompletableFuture<Boolean> result = new CompletableFuture<>(); |
| 80 | |
| 81 | SharedExecutors.getPoolThreadExecutor().submit(() -> result.complete(doDeployConfig(pipeconf))); |
| 82 | |
| 83 | return result; |
| 84 | } |
| 85 | |
| 86 | private boolean doDeployConfig(PiPipeconf pipeconf) { |
| 87 | |
| 88 | P4RuntimeController controller = handler().get(P4RuntimeController.class); |
| 89 | |
| 90 | DeviceId deviceId = handler().data().deviceId(); |
| 91 | |
| 92 | if (!controller.hasClient(deviceId)) { |
| 93 | log.warn("Unable to find client for {}, aborting pipeconf deploy", deviceId); |
| 94 | return false; |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 95 | } |
| 96 | |
| 97 | P4RuntimeClient client = controller.getClient(deviceId); |
| 98 | |
| 99 | //creating the ByteBuffer with all the needed elements to set the pipeline on the device |
| 100 | ByteBuffer pipelineBuffer = createPipelineBuffer(pipeconf, |
| 101 | ImmutableList.of(ExtensionType.TOFINO_BIN, ExtensionType.TOFINO_CONTEXT_JSON)); |
| 102 | |
| 103 | try { |
| 104 | if (!client.setPipelineConfig(pipeconf, pipelineBuffer).get()) { |
| 105 | log.warn("Unable to deploy pipeconf {} to {}", pipeconf.id(), deviceId); |
| 106 | return false; |
| 107 | } |
| 108 | |
| 109 | // It would be more logical to have this performed at device handshake, but P4runtime would reject any |
| 110 | // command if a P4info has not been set first. |
| 111 | if (!client.initStreamChannel().get()) { |
| 112 | log.warn("Unable to init stream channel to {}.", deviceId); |
| 113 | return false; |
| 114 | } |
| 115 | |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 116 | SharedExecutors.getPoolThreadExecutor().submit(() -> setupPorts(deviceId)); |
Brian O'Connor | 98fcd80 | 2017-09-13 23:40:45 -0700 | [diff] [blame] | 117 | |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 118 | } catch (InterruptedException | ExecutionException e) { |
| 119 | throw new RuntimeException(e); |
| 120 | } |
| 121 | |
| 122 | return true; |
| 123 | } |
| 124 | |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 125 | // FIXME Currently use thrift, may use gNMI to manage it in the future. |
| 126 | private void setupPorts(DeviceId deviceId) { |
| 127 | NetworkConfigService netcfgService = handler().get(NetworkConfigService.class); |
| 128 | GeneralProviderDeviceConfig providerConfig = |
| 129 | netcfgService.getConfig(deviceId, GeneralProviderDeviceConfig.class); |
| 130 | PortDescriptionsConfig portConfig = |
| 131 | netcfgService.getConfig(deviceId, PortDescriptionsConfig.class); |
Brian O'Connor | 98fcd80 | 2017-09-13 23:40:45 -0700 | [diff] [blame] | 132 | |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 133 | Map<String, String> values = |
| 134 | providerConfig.protocolsInfo().get(P4RUNTIME).configValues(); |
| 135 | String ip = values.get(PROTOCOL_KEY_IP); |
| 136 | byte tofinoDeviceId = Byte.parseByte(values.get(DEVICE_ID)); |
| 137 | |
| 138 | List<PortDescription> ports = portConfig.portDescriptions(); |
| 139 | ports.forEach(port -> { |
| 140 | pal_port_speed_t speed = getPortSpeed(port.portSpeed()); |
| 141 | addAndEnablePort(ip, tofinoDeviceId, (int) port.portNumber().toLong(), speed); |
| 142 | }); |
| 143 | } |
| 144 | |
| 145 | private pal_port_speed_t getPortSpeed(long speed) { |
| 146 | // Mbps -> BF_SPEED (1G~100G) |
| 147 | speed = speed / 1000; |
| 148 | |
| 149 | if (speed >= 100) { |
| 150 | return pal_port_speed_t.BF_SPEED_100G; |
| 151 | } |
| 152 | if (speed >= 50) { |
| 153 | return pal_port_speed_t.BF_SPEED_50G; |
| 154 | } |
| 155 | if (speed >= 40) { |
| 156 | return pal_port_speed_t.BF_SPEED_40G; |
| 157 | } |
| 158 | if (speed >= 25) { |
| 159 | return pal_port_speed_t.BF_SPEED_25G; |
| 160 | } |
| 161 | if (speed >= 10) { |
| 162 | return pal_port_speed_t.BF_SPEED_10G; |
| 163 | } |
| 164 | |
| 165 | return pal_port_speed_t.BF_SPEED_1G; |
| 166 | } |
| 167 | |
| 168 | private void addAndEnablePort(String ip, byte deviceId, int dp, pal_port_speed_t speed) { |
| 169 | TTransport transport = null; |
| 170 | try { |
| 171 | transport = new TSocket(ip, PAL_MGMT_PORT); |
| 172 | transport.open(); |
| 173 | TProtocol protocol = new TBinaryProtocol(transport); |
| 174 | TMultiplexedProtocol mProtocol = new TMultiplexedProtocol(protocol, |
| 175 | PAL_THRIFT_SERVICE); |
| 176 | pal.Client client = new pal.Client(mProtocol); |
| 177 | |
| 178 | client.pal_port_add(deviceId, dp, speed, pal_fec_type_t.BF_FEC_TYP_NONE); |
Carmelo Cascone | 254683f | 2017-12-18 17:56:27 -0800 | [diff] [blame^] | 179 | client.pal_port_an_set(deviceId, dp, pal_autoneg_policy_t.BF_AN_FORCE_DISABLE); |
Yi Tseng | 59ef170 | 2017-10-05 23:33:22 -0700 | [diff] [blame] | 180 | client.pal_port_enable(deviceId, dp); |
| 181 | transport.close(); |
| 182 | } catch (TException x) { |
| 183 | log.error("Error adding port {} to device {} ({})", dp, deviceId, ip, x); |
| 184 | } finally { |
| 185 | if (transport != null) { |
| 186 | transport.close(); |
Brian O'Connor | 98fcd80 | 2017-09-13 23:40:45 -0700 | [diff] [blame] | 187 | } |
Brian O'Connor | 98fcd80 | 2017-09-13 23:40:45 -0700 | [diff] [blame] | 188 | } |
| 189 | } |
| 190 | |
Andrea Campanella | bf1301d | 2017-08-07 18:33:52 +0200 | [diff] [blame] | 191 | private ByteBuffer createPipelineBuffer(PiPipeconf pipeconf, List<ExtensionType> targetConfigExtTypes) { |
| 192 | if (targetConfigExtTypes == null || targetConfigExtTypes.isEmpty() || isNullOrEmpty(pipeconf.id().toString())) { |
| 193 | |
| 194 | log.warn("Not enough information to deploy the Pipeconf {} on the switch {}, {}", pipeconf.id(), |
| 195 | targetConfigExtTypes); |
| 196 | } else { |
| 197 | List<ByteBuffer> buffers = Lists.newLinkedList(); |
| 198 | //Name of the pipeconf. |
| 199 | //Appears to be an arbitrary name and unrelated to p4 program |
| 200 | String name = pipeconf.id().toString(); |
| 201 | buffers.add(ByteBuffer.allocate(4 + name.length()) |
| 202 | .order(ByteOrder.LITTLE_ENDIAN) |
| 203 | .putInt(name.length()) |
| 204 | .put(name.getBytes(StandardCharsets.UTF_8))); |
| 205 | |
| 206 | // Build buffers for all the extensions needed to deploy the pipeconf to Tofino |
| 207 | targetConfigExtTypes.forEach(targetConfigExtType -> { |
| 208 | if (!pipeconf.extension(targetConfigExtType).isPresent()) { |
| 209 | // FIXME this will break the expected data format; the resulting buffer will be invalid. |
| 210 | // FIXME Consider a stronger response here, like an exception. |
| 211 | log.warn("Missing extension {} in pipeconf {}", targetConfigExtType, pipeconf.id()); |
| 212 | } |
| 213 | InputStream targetConfig = pipeconf.extension(targetConfigExtType).get(); |
| 214 | try { |
| 215 | log.info("Setting extension {} in pipeconf {}", targetConfigExtType, pipeconf.id()); |
| 216 | byte[] bin = IOUtils.toByteArray(targetConfig); |
| 217 | //length and byte of every extension |
| 218 | buffers.add(ByteBuffer.allocate(4 + bin.length) |
| 219 | .order(ByteOrder.LITTLE_ENDIAN) |
| 220 | .putInt(bin.length) |
| 221 | .put(bin)); |
| 222 | } catch (IOException ex) { |
| 223 | // FIXME this will break the expected data format; the resulting buffer will be invalid. |
| 224 | // FIXME Consider a stronger response here, like an exception. |
| 225 | log.warn("Unable to load target-specific config for {}", ex.getMessage()); |
| 226 | } |
| 227 | }); |
| 228 | |
| 229 | // Merge the buffers |
| 230 | int len = buffers.stream().mapToInt(Buffer::limit).sum(); |
| 231 | ByteBuffer deviceData = ByteBuffer.allocate(len); |
| 232 | for (ByteBuffer b : buffers) { |
| 233 | deviceData.put((ByteBuffer) b.flip()); |
| 234 | } |
| 235 | deviceData.flip(); // prepare for reading |
| 236 | return deviceData.asReadOnlyBuffer(); |
| 237 | } |
| 238 | return null; |
| 239 | } |
| 240 | |
| 241 | @Override |
| 242 | public Optional<PiPipeconf> getDefaultPipeconf() { |
| 243 | return Optional.of(DEFAULT_PIPECONF); |
| 244 | } |
| 245 | } |