apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java

/*
 * Copyright 2016-present Open Networking Laboratory
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.onosproject.openstacknetworking.routing;

import com.google.common.collect.Lists;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.Service;
import org.onlab.packet.Ethernet;
import org.onlab.packet.IPv4;
import org.onlab.packet.Ip4Address;
import org.onlab.packet.MacAddress;
import org.onlab.packet.UDP;
import org.onlab.util.KryoNamespace;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Port;
import org.onosproject.net.config.ConfigFactory;
import org.onosproject.net.config.NetworkConfigEvent;
import org.onosproject.net.config.NetworkConfigListener;
import org.onosproject.net.config.NetworkConfigRegistry;
import org.onosproject.net.config.NetworkConfigService;
import org.onosproject.net.device.DeviceEvent;
import org.onosproject.net.device.DeviceListener;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.DriverService;
import org.onosproject.net.flowobjective.FlowObjectiveService;
import org.onosproject.net.packet.InboundPacket;
import org.onosproject.net.packet.PacketContext;
import org.onosproject.net.packet.PacketProcessor;
import org.onosproject.net.packet.PacketService;
import org.onosproject.openstackinterface.OpenstackFloatingIP;
import org.onosproject.openstackinterface.OpenstackInterfaceService;
import org.onosproject.openstackinterface.OpenstackPort;
import org.onosproject.openstackinterface.OpenstackRouter;
import org.onosproject.openstackinterface.OpenstackRouterInterface;
import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
import org.onosproject.openstacknetworking.OpenstackPortInfo;
import org.onosproject.openstacknetworking.OpenstackRoutingService;
import org.onosproject.openstacknetworking.OpenstackSubjectFactories;
import org.onosproject.openstacknetworking.OpenstackSwitchingService;
import org.onosproject.store.serializers.KryoNamespaces;
import org.onosproject.store.service.ConsistentMap;
import org.onosproject.store.service.Serializer;
import org.onosproject.store.service.StorageService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;

import static com.google.common.base.Preconditions.checkNotNull;
import static org.onlab.util.Tools.groupedThreads;

@Component(immediate = true)
@Service
/**
 * Populates flow rules about L3 functionality for VMs in Openstack.
 */
public class OpenstackRoutingManager implements OpenstackRoutingService {

    private final Logger log = LoggerFactory.getLogger(getClass());

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected CoreService coreService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected PacketService packetService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected DeviceService deviceService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected OpenstackInterfaceService openstackService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected OpenstackSwitchingService openstackSwitchingService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected FlowObjectiveService flowObjectiveService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected DriverService driverService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected NetworkConfigService configService;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected NetworkConfigRegistry configRegistry;

    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
    protected StorageService storageService;


    private ApplicationId appId;
    private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
    private ConsistentMap<String, OpenstackFloatingIP> floatingIpMap; // Map<FloatingIp`s Id, FloatingIp object>
    // Map<RouterInterface`s portId, Corresponded port`s network id>
    private ConsistentMap<String, String> routerInterfaceMap;
    private static final String APP_ID = "org.onosproject.openstackrouting";
    private static final String PORT_NAME = "portName";
    private static final String PORTNAME_PREFIX_VM = "tap";
    private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
    private static final String FLOATING_IP_MAP_NAME = "openstackrouting-floatingip";
    private static final String TP_PORT_MAP_NAME = "openstackrouting-tpportnum";
    private static final String ROUTER_INTERFACE_MAP_NAME = "openstackrouting-routerinterface";
    private static final String COLON = ":";
    private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
    private static final int TP_PORT_MINIMUM_NUM = 1024;
    private static final int TP_PORT_MAXIMUM_NUM = 65535;

    private final ConfigFactory configFactory =
            new ConfigFactory(OpenstackSubjectFactories.USER_DEFINED_SUBJECT_FACTORY, OpenstackNetworkingConfig.class,
                    "config") {
                @Override
                public OpenstackNetworkingConfig createConfig() {
                    return new OpenstackNetworkingConfig();
                }
            };

    private final NetworkConfigListener configListener = new InternalConfigListener();

    private OpenstackNetworkingConfig config;
    private static final KryoNamespace.Builder FLOATING_IP_SERIALIZER = KryoNamespace.newBuilder()
            .register(KryoNamespaces.API)
            .register(KryoNamespaces.MISC)
            .register(OpenstackFloatingIP.FloatingIpStatus.class)
            .register(OpenstackFloatingIP.class)
            .register(Ip4Address.class)
            .register(String.class);

    private static final KryoNamespace.Builder NUMBER_SERIALIZER = KryoNamespace.newBuilder()
            .register(KryoNamespaces.API)
            .register(KryoNamespaces.MISC)
            .register(Integer.class)
            .register(String.class);

    private static final KryoNamespace.Builder ROUTER_INTERFACE_SERIALIZER = KryoNamespace.newBuilder()
            .register(KryoNamespaces.API)
            .register(KryoNamespaces.MISC)
            .register(String.class);

    private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
    private InternalDeviceListener internalDeviceListener = new InternalDeviceListener();
    private ExecutorService l3EventExecutorService =
            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "L3-event"));
    private ExecutorService icmpEventExecutorService =
            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "icmp-event"));
    private ExecutorService arpEventExecutorService =
            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "arp-event"));
    private OpenstackIcmpHandler openstackIcmpHandler;
    private OpenstackRoutingArpHandler openstackArpHandler;
    private OpenstackRoutingRulePopulator rulePopulator;
    private Map<DeviceId, Ip4Address> computeNodeMap;

    @Activate
    protected void activate() {
        appId = coreService.registerApplication(APP_ID);
        packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
        configRegistry.registerConfigFactory(configFactory);
        configService.addListener(configListener);
        deviceService.addListener(internalDeviceListener);

        floatingIpMap = storageService.<String, OpenstackFloatingIP>consistentMapBuilder()
                .withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
                .withName(FLOATING_IP_MAP_NAME)
                .withApplicationId(appId)
                .build();
        tpPortNumMap = storageService.<Integer, String>consistentMapBuilder()
                .withSerializer(Serializer.using(NUMBER_SERIALIZER.build()))
                .withName(TP_PORT_MAP_NAME)
                .withApplicationId(appId)
                .build();
        routerInterfaceMap = storageService.<String, String>consistentMapBuilder()
                .withSerializer(Serializer.using(ROUTER_INTERFACE_SERIALIZER.build()))
                .withName(ROUTER_INTERFACE_MAP_NAME)
                .withApplicationId(appId)
                .build();

        readConfiguration();

        log.info("started");
    }

    @Deactivate
    protected void deactivate() {
        packetService.removeProcessor(internalPacketProcessor);
        deviceService.removeListener(internalDeviceListener);
        l3EventExecutorService.shutdown();
        icmpEventExecutorService.shutdown();
        arpEventExecutorService.shutdown();

        floatingIpMap.clear();
        tpPortNumMap.clear();
        routerInterfaceMap.clear();

        log.info("stopped");
    }


    @Override
    public void createFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
        floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
    }

    @Override
    public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
        if (!floatingIpMap.containsKey(openstackFloatingIp.id())) {
            log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIp.id());
            return;
        }
        if (openstackFloatingIp.portId() == null || openstackFloatingIp.portId().equals("null")) {
            OpenstackFloatingIP floatingIp = floatingIpMap.get(openstackFloatingIp.id()).value();
            OpenstackPortInfo portInfo = openstackSwitchingService.openstackPortInfo()
                    .get(PORTNAME_PREFIX_VM.concat(floatingIp.portId().substring(0, 11)));
            if (portInfo == null) {
                log.warn("There`s no portInfo information about portId {}", floatingIp.portId());
                return;
            }
            l3EventExecutorService.execute(
                    new OpenstackFloatingIPHandler(rulePopulator, floatingIp, false, portInfo));
            floatingIpMap.replace(floatingIp.id(), openstackFloatingIp);
        } else {
            floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
            l3EventExecutorService.execute(
                    new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIp, true, null));
        }
    }

    @Override
    public void deleteFloatingIP(String id) {
        floatingIpMap.remove(id);
    }

    @Override
    public void createRouter(OpenstackRouter openstackRouter) {
    }

    @Override
    public void updateRouter(OpenstackRouter openstackRouter) {
        if (openstackRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
            checkExternalConnection(openstackRouter, getOpenstackRouterInterface(openstackRouter));
        } else {
            unsetExternalConnection();
        }
    }

    private void unsetExternalConnection() {
        Collection<OpenstackRouter> internalRouters = getExternalRouter(false);
        internalRouters.forEach(r ->
                getOpenstackRouterInterface(r).forEach(i -> rulePopulator.removeExternalRules(i)));
    }

    private Collection<OpenstackRouter> getExternalRouter(boolean externalConnection) {
        List<OpenstackRouter> routers;
        if (externalConnection) {
            routers = openstackService.routers()
                    .stream()
                    .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() > 0))
                    .collect(Collectors.toList());
        } else {
            routers = openstackService.routers()
                    .stream()
                    .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() == 0))
                    .collect(Collectors.toList());
        }
        return routers;
    }

    @Override
    public void deleteRouter(String id) {
        //TODO : In now, there`s nothing to do for deleteRouter process. It is reserved.
    }

    @Override
    public void updateRouterInterface(OpenstackRouterInterface routerInterface) {
        List<OpenstackRouterInterface> routerInterfaces = Lists.newArrayList();
        routerInterfaces.add(routerInterface);
        checkExternalConnection(getOpenstackRouter(routerInterface.id()), routerInterfaces);
        setL3Connection(getOpenstackRouter(routerInterface.id()), null);
        routerInterfaceMap.put(routerInterface.portId(), openstackService.port(routerInterface.portId()).networkId());
    }

    private void setL3Connection(OpenstackRouter openstackRouter, OpenstackPort openstackPort) {
        Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(openstackRouter);
        if (interfaceList.size() < 2) {
            return;
        }
        if (openstackPort == null) {
            interfaceList.forEach(i -> {
                OpenstackPort interfacePort = openstackService.port(i.portId());
                openstackService.ports()
                        .stream()
                        .filter(p -> p.networkId().equals(interfacePort.networkId())
                                && !p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
                        .forEach(p -> rulePopulator.populateL3Rules(p,
                                getL3ConnectionList(p.networkId(), interfaceList)));
            });
        } else {
            rulePopulator.populateL3Rules(openstackPort, getL3ConnectionList(openstackPort.networkId(), interfaceList));
        }

    }

    private List<OpenstackRouterInterface> getL3ConnectionList(String networkId,
                                                               Collection<OpenstackRouterInterface> interfaceList) {
        List<OpenstackRouterInterface> targetList = Lists.newArrayList();
        interfaceList.forEach(i -> {
            OpenstackPort port = openstackService.port(i.portId());
            if (!port.networkId().equals(networkId)) {
                targetList.add(i);
            }
        });
        return targetList;
    }

    @Override
    public void removeRouterInterface(OpenstackRouterInterface routerInterface) {
        OpenstackRouter router = openstackService.router(routerInterface.id());
        Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(router);
        if (interfaceList.size() == 1) {
            List<OpenstackRouterInterface> newList = Lists.newArrayList();
            newList.add(routerInterface);
            interfaceList.forEach(i -> removeL3RulesForRouterInterface(i, router, newList));
        }
        removeL3RulesForRouterInterface(routerInterface, router, null);
        rulePopulator.removeExternalRules(routerInterface);
        routerInterfaceMap.remove(routerInterface.portId());
    }

    private void removeL3RulesForRouterInterface(OpenstackRouterInterface routerInterface, OpenstackRouter router,
                                                 List<OpenstackRouterInterface> newList) {
        openstackService.ports(routerInterfaceMap.get(routerInterface.portId()).value()).forEach(p -> {
                    Ip4Address vmIp = (Ip4Address) p.fixedIps().values().toArray()[0];
                    if (newList == null) {
                        rulePopulator.removeL3Rules(vmIp,
                                getL3ConnectionList(p.networkId(), getOpenstackRouterInterface(router)));
                    } else {
                        rulePopulator.removeL3Rules(vmIp, newList);
                    }
                }
        );
    }

    @Override
    public void checkDisassociatedFloatingIp(String portId, OpenstackPortInfo portInfo) {
        if (floatingIpMap.size() < 1) {
            log.info("No information in FloatingIpMap");
            return;
        }
        OpenstackFloatingIP floatingIp = associatedFloatingIps()
                .stream()
                .filter(fIp -> fIp.portId().equals(portId))
                .findAny()
                .orElse(null);
        if (floatingIp != null && portInfo != null) {
            l3EventExecutorService.execute(
                    new OpenstackFloatingIPHandler(rulePopulator, floatingIp, false, portInfo));
            OpenstackFloatingIP.Builder fBuilder = new OpenstackFloatingIP.Builder()
                    .floatingIpAddress(floatingIp.floatingIpAddress())
                    .id(floatingIp.id())
                    .networkId(floatingIp.networkId())
                    .status(floatingIp.status())
                    .tenantId(floatingIp.tenantId());
            floatingIpMap.replace(floatingIp.id(), fBuilder.build());
        } else if (portInfo == null) {
            log.warn("portInfo is null as timing issue between ovs port update event and openstack deletePort event");
        }
    }

    @Override
    public String networkIdForRouterInterface(String portId) {
        return routerInterfaceMap.get(portId).value();
    }

    private Collection<OpenstackFloatingIP> associatedFloatingIps() {
        List<OpenstackFloatingIP> fIps = Lists.newArrayList();
        floatingIpMap.values()
                .stream()
                .filter(fIp -> fIp.value().portId() != null)
                .forEach(fIp -> fIps.add(fIp.value()));
        return fIps;
    }

    private void reloadInitL3Rules() {
        l3EventExecutorService.execute(() ->
                        openstackService.ports()
                                .stream()
                                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
                                .forEach(p -> updateRouterInterface(portToRouterInterface(p)))
        );

    }

    private OpenstackRouterInterface portToRouterInterface(OpenstackPort p) {
        OpenstackRouterInterface.Builder osBuilder = new OpenstackRouterInterface.Builder()
                .id(checkNotNull(p.deviceId()))
                .tenantId(checkNotNull(openstackService.network(p.networkId()).tenantId()))
                .subnetId(checkNotNull(p.fixedIps().keySet().stream().findFirst().orElse(null)).toString())
                .portId(checkNotNull(p.id()));

        return osBuilder.build();
    }

    private class InternalPacketProcessor implements PacketProcessor {

        @Override
        public void process(PacketContext context) {

            if (context.isHandled()) {
                return;
            } else if (!context.inPacket().receivedFrom().deviceId().toString()
                    .equals(config.gatewayBridgeId())) {
                return;
            }

            InboundPacket pkt = context.inPacket();
            Ethernet ethernet = pkt.parsed();

            //TODO: Considers IPv6 later.
            if (ethernet == null) {
                return;
            } else if (ethernet.getEtherType() == Ethernet.TYPE_IPV4) {
                IPv4 iPacket = (IPv4) ethernet.getPayload();
                switch (iPacket.getProtocol()) {
                    case IPv4.PROTOCOL_ICMP:

                        icmpEventExecutorService.execute(() ->
                                openstackIcmpHandler.processIcmpPacket(context, ethernet));
                        break;
                    case IPv4.PROTOCOL_UDP:
                        // don't process DHCP
                        UDP udpPacket = (UDP) iPacket.getPayload();
                        if (udpPacket.getDestinationPort() == UDP.DHCP_SERVER_PORT &&
                                udpPacket.getSourcePort() == UDP.DHCP_CLIENT_PORT) {
                            break;
                        }
                    default:
                        int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
                        Optional<Port> port =
                                getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
                        if (port.isPresent()) {
                            OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
                                    Ip4Address.valueOf(iPacket.getSourceAddress()));
                            l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
                                    portNum, openstackPort, port.get(), config));
                        } else {
                            log.warn("There`s no external interface");
                        }
                        break;
                }
            } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
                arpEventExecutorService.execute(() ->
                        openstackArpHandler.processArpPacketFromRouter(context, ethernet));
            }
        }

        private int getPortNum(MacAddress sourceMac, int destinationAddress) {
            int portNum = findUnusedPortNum();
            if (portNum == 0) {
                clearPortNumMap();
                portNum = findUnusedPortNum();
            }
            tpPortNumMap.put(portNum, sourceMac.toString().concat(COLON).concat(String.valueOf(destinationAddress)));
            return portNum;
        }

        private int findUnusedPortNum() {
            for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
                if (!tpPortNumMap.containsKey(i)) {
                    return i;
                }
            }
            return 0;
        }

    }

    private void clearPortNumMap() {
        tpPortNumMap.entrySet().forEach(e -> {
            if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
                tpPortNumMap.remove(e.getKey());
            }
        });
    }

    private Optional<Port> getExternalPort(DeviceId deviceId, String interfaceName) {
        return deviceService.getPorts(deviceId)
                .stream()
                .filter(p -> p.annotations().value(PORT_NAME).equals(interfaceName))
                .findAny();
    }

    private void checkExternalConnection(OpenstackRouter router,
                                         Collection<OpenstackRouterInterface> interfaces) {
        checkNotNull(router, "Router can not be null");
        checkNotNull(interfaces, "routerInterfaces can not be null");
        Ip4Address externalIp = router.gatewayExternalInfo().externalFixedIps()
                .values().stream().findFirst().orElse(null);
        if ((externalIp == null) || (!router.gatewayExternalInfo().isEnablePnat())) {
            log.debug("Not satisfied to set pnat configuration");
            return;
        }
        interfaces.forEach(this::initiateL3Rule);
    }

    private Optional<OpenstackRouter> getRouterfromExternalIp(Ip4Address externalIp) {
        return getExternalRouter(true)
                .stream()
                .filter(r -> r.gatewayExternalInfo()
                        .externalFixedIps()
                        .values()
                        .stream()
                        .findAny()
                        .get()
                        .equals(externalIp))
                .findAny();
    }

    private void initiateL3Rule(OpenstackRouterInterface routerInterface) {
        long vni = Long.parseLong(openstackService.network(openstackService
                .port(routerInterface.portId()).networkId()).segmentId());
        rulePopulator.populateExternalRules(vni);
    }

    private Collection<OpenstackRouterInterface> getOpenstackRouterInterface(OpenstackRouter router) {
        List<OpenstackRouterInterface> interfaces = Lists.newArrayList();
        openstackService.ports()
                .stream()
                .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
                        && p.deviceId().equals(router.id()))
                .forEach(p -> interfaces.add(portToRouterInterface(p)));
        return interfaces;
    }

    private OpenstackRouter getOpenstackRouter(String id) {
        return openstackService.routers().stream().filter(r ->
                r.id().equals(id)).iterator().next();
    }

    private OpenstackPort getOpenstackPort(MacAddress sourceMac, Ip4Address ip4Address) {
        OpenstackPort openstackPort = openstackService.ports().stream()
                .filter(p -> p.macAddress().equals(sourceMac)).iterator().next();
        return openstackPort.fixedIps().values().stream().filter(ip ->
                ip.equals(ip4Address)).count() > 0 ? openstackPort : null;
    }

    private void readConfiguration() {
        config = configService.getConfig("openstacknetworking", OpenstackNetworkingConfig.class);
        if (config == null) {
            log.error("No configuration found");
            return;
        }

        checkNotNull(config.physicalRouterMac());
        checkNotNull(config.gatewayBridgeId());
        checkNotNull(config.gatewayExternalInterfaceMac());
        checkNotNull(config.gatewayExternalInterfaceName());

        log.warn("Configured info: {}, {}, {}, {}", config.physicalRouterMac(), config.gatewayBridgeId(),
                config.gatewayExternalInterfaceMac(), config.gatewayExternalInterfaceName());

        rulePopulator = new OpenstackRoutingRulePopulator(appId,
                openstackService, flowObjectiveService, deviceService, driverService, config);

        openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService,
                openstackService, config, openstackSwitchingService);
        openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, config);

        openstackIcmpHandler.requestPacket(appId);
        openstackArpHandler.requestPacket(appId);
        reloadInitL3Rules();

        log.info("OpenstackRouting configured");
    }

    private class InternalConfigListener implements NetworkConfigListener {

        @Override
        public void event(NetworkConfigEvent event) {
            if (!event.configClass().equals(OpenstackNetworkingConfig.class)) {
                return;
            }

            if (event.type().equals(NetworkConfigEvent.Type.CONFIG_ADDED) ||
                    event.type().equals(NetworkConfigEvent.Type.CONFIG_UPDATED)) {
                l3EventExecutorService.execute(OpenstackRoutingManager.this::readConfiguration);
            }
        }
    }

    private class InternalDeviceListener implements DeviceListener {

        @Override
        public void event(DeviceEvent deviceEvent) {
            if (deviceEvent.type() == DeviceEvent.Type.PORT_UPDATED) {
                Port port = deviceEvent.port();
                OpenstackPortInfo portInfo = openstackSwitchingService.openstackPortInfo()
                        .get(port.annotations().value(PORT_NAME));
                OpenstackPort openstackPort = openstackService.port(port);
                OpenstackPort interfacePort = openstackService.ports()
                        .stream()
                        .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
                                && p.networkId().equals(openstackPort.networkId()))
                        .findAny()
                        .orElse(null);
                if (portInfo == null && openstackPort == null) {
                    log.warn("As delete event timing issue between routing and switching, Can`t delete L3 rules");
                    return;
                }
                if ((port.isEnabled()) && (interfacePort != null)) {
                    OpenstackRouterInterface routerInterface = portToRouterInterface(interfacePort);
                    l3EventExecutorService.execute(() ->
                            setL3Connection(getOpenstackRouter(routerInterface.id()), openstackPort));
                } else if (interfacePort != null) {
                    OpenstackRouterInterface routerInterface = portToRouterInterface(interfacePort);
                    l3EventExecutorService.execute(() -> rulePopulator.removeL3Rules(portInfo.ip(),
                            getL3ConnectionList(portInfo.networkId(),
                                    getOpenstackRouterInterface(getOpenstackRouter(routerInterface.id())))));
                }
            }
        }
    }

}