// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.network.ovs;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;

import javax.inject.Inject;
import javax.naming.ConfigurationException;
import javax.persistence.EntityExistsException;

import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.MessageSubscriber;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.OvsCreateTunnelAnswer;
import com.cloud.agent.api.OvsCreateTunnelCommand;
import com.cloud.agent.api.OvsDestroyBridgeCommand;
import com.cloud.agent.api.OvsDestroyTunnelCommand;
import com.cloud.agent.api.OvsFetchInterfaceAnswer;
import com.cloud.agent.api.OvsFetchInterfaceCommand;
import com.cloud.agent.api.OvsSetupBridgeCommand;
import com.cloud.agent.api.OvsVpcPhysicalTopologyConfigCommand;
import com.cloud.agent.api.OvsVpcRoutingPolicyConfigCommand;
import com.cloud.agent.manager.Commands;
import com.cloud.configuration.Config;
import com.cloud.exception.AgentUnavailableException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.network.Network;
import com.cloud.network.Networks.BroadcastDomainType;
import com.cloud.network.Networks.TrafficType;
import com.cloud.network.PhysicalNetworkTrafficType;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.network.dao.PhysicalNetworkTrafficTypeDao;
import com.cloud.network.ovs.dao.OvsTunnel;
import com.cloud.network.ovs.dao.OvsTunnelInterfaceDao;
import com.cloud.network.ovs.dao.OvsTunnelInterfaceVO;
import com.cloud.network.ovs.dao.OvsTunnelNetworkDao;
import com.cloud.network.ovs.dao.OvsTunnelNetworkVO;
import com.cloud.network.ovs.dao.VpcDistributedRouterSeqNoDao;
import com.cloud.network.ovs.dao.VpcDistributedRouterSeqNoVO;
import com.cloud.network.vpc.NetworkACLItemDao;
import com.cloud.network.vpc.NetworkACLItemVO;
import com.cloud.network.vpc.NetworkACLVO;
import com.cloud.network.vpc.VpcManager;
import com.cloud.network.vpc.VpcVO;
import com.cloud.network.vpc.dao.NetworkACLDao;
import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.StateListener;
import com.cloud.utils.fsm.StateMachine2;
import com.cloud.vm.Nic;
import com.cloud.vm.NicVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.dao.DomainRouterDao;
import com.cloud.vm.dao.NicDao;
import com.cloud.vm.dao.VMInstanceDao;

@Component
public class OvsTunnelManagerImpl extends ManagerBase implements OvsTunnelManager, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualMachine> {
    public static final Logger s_logger = Logger.getLogger(OvsTunnelManagerImpl.class.getName());

    // boolean _isEnabled;
    ScheduledExecutorService _executorPool;
    ScheduledExecutorService _cleanupExecutor;

    @Inject
    ConfigurationDao _configDao;
    @Inject
    NicDao _nicDao;
    @Inject
    HostDao _hostDao;
    @Inject
    PhysicalNetworkTrafficTypeDao _physNetTTDao;

    @Inject
    DomainRouterDao _routerDao;
    @Inject
    OvsTunnelNetworkDao _tunnelNetworkDao;
    @Inject
    OvsTunnelInterfaceDao _tunnelInterfaceDao;
    @Inject
    AgentManager _agentMgr;
    @Inject
    OvsNetworkTopologyGuru _ovsNetworkToplogyGuru;
    @Inject
    VpcDao _vpcDao;
    @Inject
    VpcManager _vpcMgr;
    @Inject
    protected VMInstanceDao _vmInstanceDao;
    @Inject
    NetworkDao _networkDao;
    @Inject
    MessageBus _messageBus;
    @Inject
    NetworkACLDao _networkACLDao;
    @Inject
    NetworkACLItemDao _networkACLItemDao;
    @Inject
    VpcDistributedRouterSeqNoDao _vpcDrSeqNoDao;

    @Override
    public boolean configure(String name, Map<String, Object> params)
            throws ConfigurationException {
        _executorPool = Executors.newScheduledThreadPool(10, new NamedThreadFactory("OVS"));
        _cleanupExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("OVS-Cleanup"));

        // register for network ACL updated for a VPC.
        _messageBus.subscribe("Network_ACL_Replaced", new NetworkAclEventsSubscriber());

        // register for VM state transition updates
        VirtualMachine.State.getStateMachine().registerListener(this);

        return true;
    }

    @DB
    protected OvsTunnelInterfaceVO createInterfaceRecord(String ip,
            String netmask, String mac, long hostId, String label) {
        OvsTunnelInterfaceVO ti = null;
        try {
            ti = new OvsTunnelInterfaceVO(ip, netmask, mac, hostId, label);
            // TODO: Is locking really necessary here?
            OvsTunnelInterfaceVO lock = _tunnelInterfaceDao
                    .acquireInLockTable(Long.valueOf(1));
            if (lock == null) {
                s_logger.warn("Cannot lock table ovs_tunnel_account");
                return null;
            }
            _tunnelInterfaceDao.persist(ti);
            _tunnelInterfaceDao.releaseFromLockTable(lock.getId());
        } catch (EntityExistsException e) {
            s_logger.debug("A record for the interface for network " + label
                    + " on host id " + hostId + " already exists");
        }
        return ti;
    }

    private String handleFetchInterfaceAnswer(Answer[] answers, Long hostId) {
        OvsFetchInterfaceAnswer ans = (OvsFetchInterfaceAnswer)answers[0];
        if (ans.getResult()) {
            if (ans.getIp() != null && !("".equals(ans.getIp()))) {
                OvsTunnelInterfaceVO ti = createInterfaceRecord(ans.getIp(),
                        ans.getNetmask(), ans.getMac(), hostId, ans.getLabel());
                return ti.getIp();
            }
        }
        // Fetch interface failed!
        s_logger.warn("Unable to fetch the IP address for the GRE tunnel endpoint"
                + ans.getDetails());
        return null;
    }

    @DB
    protected OvsTunnelNetworkVO createTunnelRecord(long from, long to, long networkId, int key) {
        OvsTunnelNetworkVO ta = null;
        try {
            ta = new OvsTunnelNetworkVO(from, to, key, networkId);
            OvsTunnelNetworkVO lock = _tunnelNetworkDao.acquireInLockTable(Long.valueOf(1));
            if (lock == null) {
                s_logger.warn("Cannot lock table ovs_tunnel_account");
                return null;
            }
            _tunnelNetworkDao.persist(ta);
            _tunnelNetworkDao.releaseFromLockTable(lock.getId());
        } catch (EntityExistsException e) {
            s_logger.debug("A record for the tunnel from " + from + " to " + to + " already exists");
        }
        return ta;
    }

    private void handleCreateTunnelAnswer(Answer[] answers) {
        OvsCreateTunnelAnswer r = (OvsCreateTunnelAnswer)answers[0];
        String s =
                String.format("(hostIP:%1$s, remoteIP:%2$s, bridge:%3$s," + "greKey:%4$s, portName:%5$s)",
                        r.getFromIp(), r.getToIp(), r.getBridge(), r.getKey(), r.getInPortName());
        Long from = r.getFrom();
        Long to = r.getTo();
        long networkId = r.getNetworkId();
        OvsTunnelNetworkVO tunnel = _tunnelNetworkDao.getByFromToNetwork(from, to, networkId);
        if (tunnel == null) {
            throw new CloudRuntimeException(
                    String.format("Unable find tunnelNetwork record" +
                            "(from=%1$s,to=%2$s, account=%3$s",
                            from, to, networkId));
        }
        if (!r.getResult()) {
            tunnel.setState(OvsTunnel.State.Failed.name());
            s_logger.warn("Create GRE tunnel from " + from + " to " + to + " failed due to " + r.getDetails()
                    + s);
        } else {
            tunnel.setState(OvsTunnel.State.Established.name());
            tunnel.setPortName(r.getInPortName());
            s_logger.info("Create GRE tunnel from " + from + " to " + to + " succeeded." + r.getDetails() + s);
        }
        _tunnelNetworkDao.update(tunnel.getId(), tunnel);
    }

    private String getGreEndpointIP(Host host, Network nw)
            throws AgentUnavailableException, OperationTimedoutException {
        String endpointIp = null;
        // Fetch fefault name for network label from configuration
        String physNetLabel = _configDao.getValue(Config.OvsTunnelNetworkDefaultLabel.key());
        Long physNetId = nw.getPhysicalNetworkId();
        PhysicalNetworkTrafficType physNetTT =
                _physNetTTDao.findBy(physNetId, TrafficType.Guest);
        HypervisorType hvType = host.getHypervisorType();

        String label = null;
        switch (hvType) {
        case XenServer:
            label = physNetTT.getXenNetworkLabel();
            if ((label != null) && (!label.equals(""))) {
                physNetLabel = label;
            }
            break;
        case KVM:
            label = physNetTT.getKvmNetworkLabel();
            if ((label != null) && (!label.equals(""))) {
                physNetLabel = label;
            }
            break;
        default:
            throw new CloudRuntimeException("Hypervisor " +
                    hvType.toString() +
                    " unsupported by OVS Tunnel Manager");
        }

        // Try to fetch GRE endpoint IP address for cloud db
        // If not found, then find it on the hypervisor
        OvsTunnelInterfaceVO tunnelIface =
                _tunnelInterfaceDao.getByHostAndLabel(host.getId(),
                        physNetLabel);
        if (tunnelIface == null) {
            //Now find and fetch configuration for physical interface
            //for network with label on target host
            Commands fetchIfaceCmds =
                    new Commands(new OvsFetchInterfaceCommand(physNetLabel));
            s_logger.debug("Ask host " + host.getId() +
                    " to retrieve interface for phy net with label:" +
                    physNetLabel);
            Answer[] fetchIfaceAnswers = _agentMgr.send(host.getId(), fetchIfaceCmds);
            //And finally save it for future use
            endpointIp = handleFetchInterfaceAnswer(fetchIfaceAnswers, host.getId());
        } else {
            endpointIp = tunnelIface.getIp();
        }
        return endpointIp;
    }

    private int getGreKey(Network network) {
        int key = 0;
        try {
            //The GRE key is actually in the host part of the URI
            String keyStr = network.getBroadcastUri().getAuthority();
            if (keyStr.contains(".")) {
                String[] parts = keyStr.split("\\.");
                key = Integer.parseInt(parts[1]);
            } else {
                key = Integer.parseInt(keyStr);
            }

            return key;
        } catch (NumberFormatException e) {
            s_logger.debug("Well well, how did '" + key
                    + "' end up in the broadcast URI for the network?");
            throw new CloudRuntimeException(String.format(
                    "Invalid GRE key parsed from"
                            + "network broadcast URI (%s)", network
                            .getBroadcastUri().toString()));
        }
    }

    @DB
    protected void checkAndCreateTunnel(Network nw, Host host) {

        s_logger.debug("Creating tunnels with OVS tunnel manager");

        long hostId = host.getId();
        int key = getGreKey(nw);
        String bridgeName = generateBridgeName(nw, key);
        List<Long> toHostIds = new ArrayList<Long>();
        List<Long> fromHostIds = new ArrayList<Long>();
        List<Long> networkSpannedHosts = _ovsNetworkToplogyGuru.getNetworkSpanedHosts(nw.getId());
        for (Long rh : networkSpannedHosts) {
            if (rh == hostId) {
                continue;
            }
            OvsTunnelNetworkVO ta = _tunnelNetworkDao.getByFromToNetwork(hostId, rh.longValue(), nw.getId());
            // Try and create the tunnel even if a previous attempt failed
            if (ta == null || ta.getState().equals(OvsTunnel.State.Failed.name())) {
                s_logger.debug("Attempting to create tunnel from:" + hostId + " to:" + rh.longValue());
                if (ta == null) {
                    createTunnelRecord(hostId, rh.longValue(), nw.getId(), key);
                }
                if (!toHostIds.contains(rh)) {
                    toHostIds.add(rh);
                }
            }

            ta = _tunnelNetworkDao.getByFromToNetwork(rh.longValue(),
                    hostId, nw.getId());
            // Try and create the tunnel even if a previous attempt failed
            if (ta == null || ta.getState().equals(OvsTunnel.State.Failed.name())) {
                s_logger.debug("Attempting to create tunnel from:" +
                        rh.longValue() + " to:" + hostId);
                if (ta == null) {
                    createTunnelRecord(rh.longValue(), hostId,
                            nw.getId(), key);
                }
                if (!fromHostIds.contains(rh)) {
                    fromHostIds.add(rh);
                }
            }
        }
        //TODO: Should we propagate the exception here?
        try {
            String myIp = getGreEndpointIP(host, nw);
            if (myIp == null)
                throw new GreTunnelException("Unable to retrieve the source " + "endpoint for the GRE tunnel." + "Failure is on host:" + host.getId());
            boolean noHost = true;
            for (Long i : toHostIds) {
                HostVO rHost = _hostDao.findById(i);
                String otherIp = getGreEndpointIP(rHost, nw);
                if (otherIp == null)
                    throw new GreTunnelException(
                            "Unable to retrieve the remote "
                                    + "endpoint for the GRE tunnel."
                                    + "Failure is on host:" + rHost.getId());
                Commands cmds = new Commands(
                        new OvsCreateTunnelCommand(otherIp, key,
                                Long.valueOf(hostId), i, nw.getId(), myIp, bridgeName, nw.getUuid()));
                s_logger.debug("Attempting to create tunnel from:" + hostId + " to:" + i + " for the network " + nw.getId());
                s_logger.debug("Ask host " + hostId
                        + " to create gre tunnel to " + i);
                Answer[] answers = _agentMgr.send(hostId, cmds);
                handleCreateTunnelAnswer(answers);
                noHost = false;
            }

            for (Long i : fromHostIds) {
                HostVO rHost = _hostDao.findById(i);
                String otherIp = getGreEndpointIP(rHost, nw);
                Commands cmds = new Commands(new OvsCreateTunnelCommand(myIp,
                        key, i, Long.valueOf(hostId), nw.getId(), otherIp, bridgeName, nw.getUuid()));
                s_logger.debug("Ask host " + i + " to create gre tunnel to "
                        + hostId);
                Answer[] answers = _agentMgr.send(i, cmds);
                handleCreateTunnelAnswer(answers);
                noHost = false;
            }

            // If no tunnels have been configured, perform the bridge setup
            // anyway. This will ensure VIF rules will be triggered
            if (noHost) {
                Commands cmds = new Commands(new OvsSetupBridgeCommand(bridgeName, hostId, nw.getId()));
                s_logger.debug("Ask host " + hostId + " to configure bridge for network:" + nw.getId());
                Answer[] answers = _agentMgr.send(hostId, cmds);
                handleSetupBridgeAnswer(answers);
            }
        } catch (GreTunnelException | OperationTimedoutException | AgentUnavailableException e) {
            // I really thing we should do a better handling of these exceptions
            s_logger.warn("Ovs Tunnel network created tunnel failed", e);
        }
    }

    @Override
    public boolean isOvsTunnelEnabled() {
        return true;
    }

    boolean isVpcEnabledForDistributedRouter(long vpcId) {
        VpcVO vpc = _vpcDao.findById(vpcId);
        return vpc.usesDistributedRouter();
    }

    @Override
    public void checkAndPrepareHostForTunnelNetwork(Network nw, Host host) {
        if (nw.getVpcId() != null && isVpcEnabledForDistributedRouter(nw.getVpcId())) {
            // check and setup host to be in full tunnel mesh with each of the network in the VPC
            checkAndCreateVpcTunnelNetworks(host, nw.getVpcId());
        } else {
            // check and setup host to be in full tunnel mesh with the network
            checkAndCreateTunnel(nw, host);
        }
    }

    @DB
    private void handleDestroyTunnelAnswer(Answer ans, long from, long to, long networkId) {
        if (ans.getResult()) {
            OvsTunnelNetworkVO lock = _tunnelNetworkDao.acquireInLockTable(Long.valueOf(1));
            if (lock == null) {
                s_logger.warn(String.format("failed to lock" +
                        "ovs_tunnel_account, remove record of " +
                        "tunnel(from=%1$s, to=%2$s account=%3$s) failed",
                        from, to, networkId));
                return;
            }

            _tunnelNetworkDao.removeByFromToNetwork(from, to, networkId);
            _tunnelNetworkDao.releaseFromLockTable(lock.getId());

            s_logger.debug(String.format("Destroy tunnel(account:%1$s," +
                    "from:%2$s, to:%3$s) successful",
                    networkId, from, to));
        } else {
            s_logger.debug(String.format("Destroy tunnel(account:%1$s," + "from:%2$s, to:%3$s) failed", networkId, from, to));
        }
    }

    @DB
    private void handleDestroyBridgeAnswer(Answer ans, long hostId, long networkId) {

        if (ans.getResult()) {
            OvsTunnelNetworkVO lock = _tunnelNetworkDao.acquireInLockTable(Long.valueOf(1));
            if (lock == null) {
                s_logger.warn("failed to lock ovs_tunnel_network," + "remove record");
                return;
            }

            _tunnelNetworkDao.removeByFromNetwork(hostId, networkId);
            _tunnelNetworkDao.releaseFromLockTable(lock.getId());

            s_logger.debug(String.format("Destroy bridge for" +
                    "network %1$s successful", networkId));
        } else {
            s_logger.debug(String.format("Destroy bridge for" +
                    "network %1$s failed", networkId));
        }
    }

    private void handleSetupBridgeAnswer(Answer[] answers) {
        //TODO: Add some error management here?
        s_logger.debug("Placeholder for something more meanginful to come");
    }

    @Override
    public void checkAndRemoveHostFromTunnelNetwork(Network nw, Host host) {

        if (nw.getVpcId() != null && isVpcEnabledForDistributedRouter(nw.getVpcId())) {
            List<Long> vmIds = _ovsNetworkToplogyGuru.getActiveVmsInVpcOnHost(nw.getVpcId(), host.getId());

            if (vmIds != null && !vmIds.isEmpty()) {
                return;
            }

            // there are not active VM's on this host belonging to any of the tiers in the VPC, so remove
            // the host from the tunnel mesh network and destroy the bridge
            List<? extends Network> vpcNetworks =  _vpcMgr.getVpcNetworks(nw.getVpcId());
            try {
                for (Network network: vpcNetworks) {
                    int key = getGreKey(nw);
                    String bridgeName = generateBridgeName(nw, key);
                    /* Then ask hosts have peer tunnel with me to destroy them */
                    List<OvsTunnelNetworkVO> peers = _tunnelNetworkDao.listByToNetwork(host.getId(),nw.getId());
                    for (OvsTunnelNetworkVO p : peers) {
                        // If the tunnel was not successfully created don't bother to remove it
                        if (p.getState().equals(OvsTunnel.State.Established.name())) {
                            Command cmd= new OvsDestroyTunnelCommand(p.getNetworkId(), bridgeName,
                                    p.getPortName());
                            s_logger.debug("Destroying tunnel to " + host.getId() +
                                    " from " + p.getFrom());
                            Answer ans = _agentMgr.send(p.getFrom(), cmd);
                            handleDestroyTunnelAnswer(ans, p.getFrom(), p.getTo(), p.getNetworkId());
                        }
                    }
                }

                Command cmd = new OvsDestroyBridgeCommand(nw.getId(), generateBridgeNameForVpc(nw.getVpcId()),
                        host.getId());
                s_logger.debug("Destroying bridge for network " + nw.getId() + " on host:" + host.getId());
                Answer ans = _agentMgr.send(host.getId(), cmd);
                handleDestroyBridgeAnswer(ans, host.getId(), nw.getId());
            } catch (Exception e) {
                s_logger.info("[ignored]"
                        + "exception while removing host from networks: " + e.getLocalizedMessage());
            }
        } else {
            List<Long> vmIds = _ovsNetworkToplogyGuru.getActiveVmsInNetworkOnHost(nw.getId(), host.getId(), true);
            if (vmIds != null && !vmIds.isEmpty()) {
                return;
            }
            try {
                /* Now we are last one on host, destroy the bridge with all
                * the tunnels for this network  */
                int key = getGreKey(nw);
                String bridgeName = generateBridgeName(nw, key);
                Command cmd = new OvsDestroyBridgeCommand(nw.getId(), bridgeName, host.getId());
                s_logger.debug("Destroying bridge for network " + nw.getId() + " on host:" + host.getId());
                Answer ans = _agentMgr.send(host.getId(), cmd);
                handleDestroyBridgeAnswer(ans, host.getId(), nw.getId());

                /* Then ask hosts have peer tunnel with me to destroy them */
                List<OvsTunnelNetworkVO> peers =
                        _tunnelNetworkDao.listByToNetwork(host.getId(),
                                nw.getId());
                for (OvsTunnelNetworkVO p : peers) {
                    // If the tunnel was not successfully created don't bother to remove it
                    if (p.getState().equals(OvsTunnel.State.Established.name())) {
                        cmd = new OvsDestroyTunnelCommand(p.getNetworkId(), bridgeName,
                                p.getPortName());
                        s_logger.debug("Destroying tunnel to " + host.getId() +
                                " from " + p.getFrom());
                        ans = _agentMgr.send(p.getFrom(), cmd);
                        handleDestroyTunnelAnswer(ans, p.getFrom(),
                                p.getTo(), p.getNetworkId());
                    }
                }
            } catch (Exception e) {
                s_logger.warn("Destroy tunnel failed", e);
            }
        }
    }

    private String generateBridgeName(Network nw, int key) {
        if (nw.getVpcId() != null && isVpcEnabledForDistributedRouter(nw.getVpcId())) {
            return "OVS-DR-VPC-Bridge" + nw.getVpcId();
        } else {
            return "OVSTunnel"+key;
        }
    }
    private String generateBridgeNameForVpc(long vpcId) {
        return "OVS-DR-VPC-Bridge" + vpcId;
    }

    @DB
    protected void checkAndCreateVpcTunnelNetworks(Host host, long vpcId) {

        long hostId = host.getId();
        String bridgeName=generateBridgeNameForVpc(vpcId);

        List<Long> vmIds = _ovsNetworkToplogyGuru.getActiveVmsInVpcOnHost(vpcId, hostId);

        if (vmIds == null || vmIds.isEmpty()) {

            // since this is the first VM from the VPC being launched on the host, first setup the bridge
            try {
                Commands cmds = new Commands(new OvsSetupBridgeCommand(bridgeName, hostId, null));
                s_logger.debug("Ask host " + hostId + " to create bridge for vpc " + vpcId + " and configure the "
                        + " bridge for distributed routing.");
                Answer[] answers = _agentMgr.send(hostId, cmds);
                handleSetupBridgeAnswer(answers);
            } catch (OperationTimedoutException | AgentUnavailableException e) {
                s_logger.warn("Ovs Tunnel network created tunnel failed", e);
            }

            // now that bridge is setup, populate network acl's before the VM gets created
            OvsVpcRoutingPolicyConfigCommand cmd = prepareVpcRoutingPolicyUpdate(vpcId);
            cmd.setSequenceNumber(getNextRoutingPolicyUpdateSequenceNumber(vpcId));

            if (!sendVpcRoutingPolicyChangeUpdate(cmd, hostId, bridgeName)) {
                s_logger.debug("Failed to send VPC routing policy change update to host : " + hostId +
                        ". But moving on with sending the updates to the rest of the hosts.");
            }
        }

        List<? extends Network> vpcNetworks =  _vpcMgr.getVpcNetworks(vpcId);
        List<Long> vpcSpannedHostIds = _ovsNetworkToplogyGuru.getVpcSpannedHosts(vpcId);
        for (Network vpcNetwork: vpcNetworks) {
            if (vpcNetwork.getState() != Network.State.Implemented &&
                    vpcNetwork.getState() != Network.State.Implementing && vpcNetwork.getState() != Network.State.Setup)
                continue;

            int key = getGreKey(vpcNetwork);
            List<Long> toHostIds = new ArrayList<Long>();
            List<Long> fromHostIds = new ArrayList<Long>();
            OvsTunnelNetworkVO tunnelRecord = null;

            for (Long rh : vpcSpannedHostIds) {
                if (rh == hostId) {
                    continue;
                }
                tunnelRecord = _tunnelNetworkDao.getByFromToNetwork(hostId, rh.longValue(), vpcNetwork.getId());
                // Try and create the tunnel if does not exit or previous attempt failed
                if (tunnelRecord == null || tunnelRecord.getState().equals(OvsTunnel.State.Failed.name())) {
                    s_logger.debug("Attempting to create tunnel from:" + hostId + " to:" + rh.longValue());
                    if (tunnelRecord == null) {
                        createTunnelRecord(hostId, rh.longValue(), vpcNetwork.getId(), key);
                    }
                    if (!toHostIds.contains(rh)) {
                        toHostIds.add(rh);
                    }
                }
                tunnelRecord = _tunnelNetworkDao.getByFromToNetwork(rh.longValue(), hostId, vpcNetwork.getId());
                // Try and create the tunnel if does not exit or previous attempt failed
                if (tunnelRecord == null || tunnelRecord.getState().equals(OvsTunnel.State.Failed.name())) {
                    s_logger.debug("Attempting to create tunnel from:" + rh.longValue() + " to:" + hostId);
                    if (tunnelRecord == null) {
                        createTunnelRecord(rh.longValue(), hostId, vpcNetwork.getId(), key);
                    }
                    if (!fromHostIds.contains(rh)) {
                        fromHostIds.add(rh);
                    }
                }
            }

            try {
                String myIp = getGreEndpointIP(host, vpcNetwork);
                if (myIp == null)
                    throw new GreTunnelException("Unable to retrieve the source " + "endpoint for the GRE tunnel."
                            + "Failure is on host:" + host.getId());
                boolean noHost = true;

                for (Long i : toHostIds) {
                    HostVO rHost = _hostDao.findById(i);
                    String otherIp = getGreEndpointIP(rHost, vpcNetwork);
                    if (otherIp == null)
                        throw new GreTunnelException(
                                "Unable to retrieve the remote endpoint for the GRE tunnel."
                                        + "Failure is on host:" + rHost.getId());
                    Commands cmds = new Commands( new OvsCreateTunnelCommand(otherIp, key, Long.valueOf(hostId),
                                     i, vpcNetwork.getId(), myIp, bridgeName, vpcNetwork.getUuid()));
                    s_logger.debug("Attempting to create tunnel from:" + hostId + " to:" + i + " for the network "
                            + vpcNetwork.getId());
                    s_logger.debug("Ask host " + hostId
                            + " to create gre tunnel to " + i);
                    Answer[] answers = _agentMgr.send(hostId, cmds);
                    handleCreateTunnelAnswer(answers);
                }

                for (Long i : fromHostIds) {
                    HostVO rHost = _hostDao.findById(i);
                    String otherIp = getGreEndpointIP(rHost, vpcNetwork);
                    Commands cmds = new Commands(new OvsCreateTunnelCommand(myIp,
                            key, i, Long.valueOf(hostId), vpcNetwork.getId(), otherIp, bridgeName,
                            vpcNetwork.getUuid()));
                    s_logger.debug("Ask host " + i + " to create gre tunnel to "
                            + hostId);
                    Answer[] answers = _agentMgr.send(i, cmds);
                    handleCreateTunnelAnswer(answers);
                }
            } catch (GreTunnelException | OperationTimedoutException | AgentUnavailableException e) {
                // I really thing we should do a better handling of these exceptions
                s_logger.warn("Ovs Tunnel network created tunnel failed", e);
            }
        }
    }

    @Override
    public boolean preStateTransitionEvent(VirtualMachine.State oldState,
                                           VirtualMachine.Event event, VirtualMachine.State newState,
                                           VirtualMachine vo, boolean status, Object opaque) {
        return true;
    }

    @Override
    public boolean postStateTransitionEvent(StateMachine2.Transition<VirtualMachine.State, VirtualMachine.Event> transition, VirtualMachine vm, boolean status, Object opaque) {
      if (!status) {
        return false;
      }

      VirtualMachine.State oldState = transition.getCurrentState();
      VirtualMachine.State newState = transition.getToState();
      VirtualMachine.Event event = transition.getEvent();
      if (VirtualMachine.State.isVmStarted(oldState, event, newState)) {
        handleVmStateChange((VMInstanceVO)vm);
      } else if (VirtualMachine.State.isVmStopped(oldState, event, newState)) {
        handleVmStateChange((VMInstanceVO)vm);
      } else if (VirtualMachine.State.isVmMigrated(oldState, event, newState)) {
        handleVmStateChange((VMInstanceVO)vm);
      }

      return true;
    }

  private void handleVmStateChange(VMInstanceVO vm) {

        // get the VPC's impacted with the VM start
        List<Long> vpcIds = _ovsNetworkToplogyGuru.getVpcIdsVmIsPartOf(vm.getId());
        if (vpcIds == null || vpcIds.isEmpty()) {
            return;
        }

        for (Long vpcId: vpcIds) {
            VpcVO vpc = _vpcDao.findById(vpcId);
            // nothing to do if the VPC is not setup for distributed routing
            if (vpc == null || !vpc.usesDistributedRouter()) {
                return;
            }

            // get the list of hosts on which VPC spans (i.e hosts that need to be aware of VPC topology change update)
            List<Long> vpcSpannedHostIds = _ovsNetworkToplogyGuru.getVpcSpannedHosts(vpcId);
            String bridgeName=generateBridgeNameForVpc(vpcId);

            OvsVpcPhysicalTopologyConfigCommand topologyConfigCommand = prepareVpcTopologyUpdate(vpcId);
            topologyConfigCommand.setSequenceNumber(getNextTopologyUpdateSequenceNumber(vpcId));

            // send topology change update to VPC spanned hosts
            for (Long id: vpcSpannedHostIds) {
                if (!sendVpcTopologyChangeUpdate(topologyConfigCommand, id, bridgeName)) {
                    s_logger.debug("Failed to send VPC topology change update to host : " + id + ". Moving on " +
                            "with rest of the host update.");
                }
            }
        }
    }

    public boolean sendVpcTopologyChangeUpdate(OvsVpcPhysicalTopologyConfigCommand updateCmd, long hostId, String bridgeName) {
        try {
            s_logger.debug("Sending VPC topology change update to the host " + hostId);
            updateCmd.setHostId(hostId);
            updateCmd.setBridgeName(bridgeName);
            Answer ans = _agentMgr.send(hostId, updateCmd);
            if (ans.getResult()) {
                s_logger.debug("Successfully updated the host " + hostId + " with latest VPC topology." );
                return true;
            }  else {
                s_logger.debug("Failed to update the host " + hostId + " with latest VPC topology." );
                return false;
            }
        } catch (Exception e) {
            s_logger.debug("Failed to updated the host " + hostId + " with latest VPC topology.", e );
            return false;
        }
    }

    OvsVpcPhysicalTopologyConfigCommand prepareVpcTopologyUpdate(long vpcId) {
        VpcVO vpc = _vpcDao.findById(vpcId);
        assert (vpc != null): "invalid vpc id";

        List<? extends Network> vpcNetworks =  _vpcMgr.getVpcNetworks(vpcId);
        List<Long> hostIds = _ovsNetworkToplogyGuru.getVpcSpannedHosts(vpcId);
        List<Long> vmIds = _ovsNetworkToplogyGuru.getAllActiveVmsInVpc(vpcId);

        List<OvsVpcPhysicalTopologyConfigCommand.Host> hosts = new ArrayList<>();
        List<OvsVpcPhysicalTopologyConfigCommand.Tier> tiers = new ArrayList<>();
        List<OvsVpcPhysicalTopologyConfigCommand.Vm> vms = new ArrayList<>();

        for (Long hostId : hostIds) {
            HostVO hostDetails = _hostDao.findById(hostId);
            String remoteIp = null;
            for (Network network: vpcNetworks) {
                try {
                    remoteIp = getGreEndpointIP(hostDetails, network);
                } catch (Exception e) {
                    s_logger.info("[ignored]"
                            + "error getting GRE endpoint: " + e.getLocalizedMessage());
                }
            }
            OvsVpcPhysicalTopologyConfigCommand.Host host = new OvsVpcPhysicalTopologyConfigCommand.Host(hostId, remoteIp);
            hosts.add(host);
        }

        for (Network network: vpcNetworks) {
            String key = network.getBroadcastUri().getAuthority();
            long gre_key;
            if (key.contains(".")) {
                String[] parts = key.split("\\.");
                gre_key = Long.parseLong(parts[1]);
            } else {
                try {
                    gre_key = Long.parseLong(BroadcastDomainType.getValue(key));
                } catch (Exception e) {
                    return null;
                }
            }
            NicVO nic = _nicDao.findByIp4AddressAndNetworkId(network.getGateway(), network.getId());
            OvsVpcPhysicalTopologyConfigCommand.Tier tier = new OvsVpcPhysicalTopologyConfigCommand.Tier(gre_key,
                    network.getUuid(), network.getGateway(), nic.getMacAddress(), network.getCidr());
            tiers.add(tier);
        }

        for (long vmId: vmIds) {
            VirtualMachine vmInstance = _vmInstanceDao.findById(vmId);
            List<OvsVpcPhysicalTopologyConfigCommand.Nic>  vmNics = new ArrayList<OvsVpcPhysicalTopologyConfigCommand.Nic>();
            for (Nic vmNic :_nicDao.listByVmId(vmId)) {
                Network network = _networkDao.findById(vmNic.getNetworkId());
                if (network.getTrafficType() == TrafficType.Guest) {
                    OvsVpcPhysicalTopologyConfigCommand.Nic nic =  new OvsVpcPhysicalTopologyConfigCommand.Nic(
                            vmNic.getIPv4Address(), vmNic.getMacAddress(), network.getUuid());
                    vmNics.add(nic);
                }
            }
            OvsVpcPhysicalTopologyConfigCommand.Vm vm = new OvsVpcPhysicalTopologyConfigCommand.Vm(
                    vmInstance.getHostId(), vmNics.toArray(new OvsVpcPhysicalTopologyConfigCommand.Nic[vmNics.size()]));
            vms.add(vm);
        }

        return new OvsVpcPhysicalTopologyConfigCommand(
                hosts.toArray(new OvsVpcPhysicalTopologyConfigCommand.Host[hosts.size()]),
                tiers.toArray(new OvsVpcPhysicalTopologyConfigCommand.Tier[tiers.size()]),
                vms.toArray(new OvsVpcPhysicalTopologyConfigCommand.Vm[vms.size()]),
                vpc.getCidr());
    }

    // Subscriber to ACL replace events. On acl replace event, if the vpc for the tier is enabled for
    // distributed routing send the ACL update to all the hosts on which VPC spans
    public class NetworkAclEventsSubscriber implements MessageSubscriber {
        @Override
        public void onPublishMessage(String senderAddress, String subject, Object args) {
            try {
                NetworkVO network = (NetworkVO) args;
                String bridgeName=generateBridgeNameForVpc(network.getVpcId());
                if (network.getVpcId() != null && isVpcEnabledForDistributedRouter(network.getVpcId())) {
                    long vpcId = network.getVpcId();
                    OvsVpcRoutingPolicyConfigCommand cmd = prepareVpcRoutingPolicyUpdate(vpcId);
                    cmd.setSequenceNumber(getNextRoutingPolicyUpdateSequenceNumber(vpcId));

                    // get the list of hosts on which VPC spans (i.e hosts that need to be aware of VPC
                    // network ACL update)
                    List<Long> vpcSpannedHostIds = _ovsNetworkToplogyGuru.getVpcSpannedHosts(vpcId);
                    for (Long id: vpcSpannedHostIds) {
                        if (!sendVpcRoutingPolicyChangeUpdate(cmd, id, bridgeName)) {
                            s_logger.debug("Failed to send VPC routing policy change update to host : " + id +
                                    ". But moving on with sending the updates to the rest of the hosts.");
                        }
                    }
                }
            } catch (Exception e) {
                s_logger.debug("Failed to send VPC routing policy change updates all hosts in vpc", e);
            }
        }
    }

    private OvsVpcRoutingPolicyConfigCommand prepareVpcRoutingPolicyUpdate(long vpcId) {

        List<OvsVpcRoutingPolicyConfigCommand.Acl> acls = new ArrayList<>();
        List<OvsVpcRoutingPolicyConfigCommand.Tier> tiers = new ArrayList<>();

        VpcVO vpc = _vpcDao.findById(vpcId);
        List<? extends Network> vpcNetworks =  _vpcMgr.getVpcNetworks(vpcId);
        assert (vpc != null && (vpcNetworks != null && !vpcNetworks.isEmpty())): "invalid vpc id";

        for (Network network : vpcNetworks) {
            Long networkAclId = network.getNetworkACLId();
            if (networkAclId == null)
                continue;
            NetworkACLVO networkAcl = _networkACLDao.findById(networkAclId);

            List<OvsVpcRoutingPolicyConfigCommand.AclItem> aclItems = new ArrayList<>();
            List<NetworkACLItemVO> aclItemVos = _networkACLItemDao.listByACL(networkAclId);
            for (NetworkACLItemVO aclItem : aclItemVos) {
                String[] sourceCidrs = aclItem.getSourceCidrList().toArray(new String[aclItem.getSourceCidrList().size()]);

                aclItems.add(new OvsVpcRoutingPolicyConfigCommand.AclItem(
                        aclItem.getNumber(), aclItem.getUuid(), aclItem.getAction().name(),
                        aclItem.getTrafficType().name(),
                        ((aclItem.getSourcePortStart() != null) ?aclItem.getSourcePortStart().toString() :null),
                        ((aclItem.getSourcePortEnd() != null) ?aclItem.getSourcePortEnd().toString() :null),
                        aclItem.getProtocol(),
                        sourceCidrs));
            }

            OvsVpcRoutingPolicyConfigCommand.Acl acl = new OvsVpcRoutingPolicyConfigCommand.Acl(networkAcl.getUuid(),
                    aclItems.toArray(new OvsVpcRoutingPolicyConfigCommand.AclItem[aclItems.size()]));
            acls.add(acl);

            OvsVpcRoutingPolicyConfigCommand.Tier tier = new OvsVpcRoutingPolicyConfigCommand.Tier(network.getUuid(),
                    network.getCidr(), networkAcl.getUuid());
            tiers.add(tier);
        }

        OvsVpcRoutingPolicyConfigCommand cmd = new OvsVpcRoutingPolicyConfigCommand(vpc.getUuid(), vpc.getCidr(),
                acls.toArray(new OvsVpcRoutingPolicyConfigCommand.Acl[acls.size()]),
                tiers.toArray(new OvsVpcRoutingPolicyConfigCommand.Tier[tiers.size()]));
        return cmd;
    }

    private boolean sendVpcRoutingPolicyChangeUpdate(OvsVpcRoutingPolicyConfigCommand updateCmd, long hostId, String bridgeName) {
        try {
            s_logger.debug("Sending VPC routing policies change update to the host " + hostId);
            updateCmd.setHostId(hostId);
            updateCmd.setBridgeName(bridgeName);
            Answer ans = _agentMgr.send(hostId, updateCmd);
            if (ans.getResult()) {
                s_logger.debug("Successfully updated the host " + hostId + " with latest VPC routing policies." );
                return true;
            }  else {
                s_logger.debug("Failed to update the host " + hostId + " with latest routing policies." );
                return false;
            }
        } catch (Exception e) {
            s_logger.debug("Failed to updated the host " + hostId + " with latest routing policies due to" , e );
            return false;
        }
    }

    private long getNextTopologyUpdateSequenceNumber(final long vpcId) {

        try {
            return  Transaction.execute(new TransactionCallback<Long>() {
                @Override
                public Long doInTransaction(TransactionStatus status) {
                    VpcDistributedRouterSeqNoVO seqVo = _vpcDrSeqNoDao.findByVpcId(vpcId);
                    if (seqVo == null) {
                        seqVo = new VpcDistributedRouterSeqNoVO(vpcId);
                        _vpcDrSeqNoDao.persist(seqVo);
                    }
                    seqVo = _vpcDrSeqNoDao.lockRow(seqVo.getId(), true);
                    seqVo.incrTopologyUpdateSequenceNo();
                    _vpcDrSeqNoDao.update(seqVo.getId(), seqVo);
                    return seqVo.getTopologyUpdateSequenceNo();
                }
            });
        } finally {

        }
    }

    private long getNextRoutingPolicyUpdateSequenceNumber(final long vpcId) {

        try {
            return  Transaction.execute(new TransactionCallback<Long>() {
                @Override
                public Long doInTransaction(TransactionStatus status) {
                    VpcDistributedRouterSeqNoVO seqVo = _vpcDrSeqNoDao.findByVpcId(vpcId);
                    if (seqVo == null) {
                        seqVo = new VpcDistributedRouterSeqNoVO(vpcId);
                        _vpcDrSeqNoDao.persist(seqVo);
                    }
                    seqVo = _vpcDrSeqNoDao.lockRow(seqVo.getId(), true);
                    seqVo.incrPolicyUpdateSequenceNo();
                    _vpcDrSeqNoDao.update(seqVo.getId(), seqVo);
                    return seqVo.getPolicyUpdateSequenceNo();
                }
            });
        } finally {

        }
    }
}
