// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.cluster;

import java.io.IOException;
import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.nio.channels.SocketChannel;
import java.rmi.RemoteException;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.SQLNonTransientException;
import java.sql.SQLRecoverableException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import javax.inject.Inject;
import javax.naming.ConfigurationException;

import org.apache.cloudstack.framework.config.ConfigDepot;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.log4j.Logger;

import com.cloud.cluster.dao.ManagementServerHostDao;
import com.cloud.cluster.dao.ManagementServerHostPeerDao;
import com.cloud.utils.DateUtil;
import com.cloud.utils.Profiler;
import com.cloud.utils.component.ComponentLifecycle;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.ConnectionConcierge;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.DbProperties;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.events.SubscriptionMgr;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.exception.ExceptionUtil;
import com.cloud.utils.mgmt.JmxUtil;
import com.cloud.utils.net.NetUtils;

public class ClusterManagerImpl extends ManagerBase implements ClusterManager, Configurable {
    private static final Logger s_logger = Logger.getLogger(ClusterManagerImpl.class);

    private static final int EXECUTOR_SHUTDOWN_TIMEOUT = 1000; // 1 second
    private static final int DEFAULT_OUTGOING_WORKERS = 5;

    private final List<ClusterManagerListener> _listeners = new ArrayList<ClusterManagerListener>();
    private final Map<Long, ManagementServerHostVO> _activePeers = new HashMap<Long, ManagementServerHostVO>();

    private final Map<String, ClusterService> _clusterPeers;

    @Inject
    protected ConfigDepot _configDepot;

    private final ScheduledExecutorService _heartbeatScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Cluster-Heartbeat"));
    private final ExecutorService _notificationExecutor = Executors.newFixedThreadPool(1, new NamedThreadFactory("Cluster-Notification"));
    private final List<ClusterManagerMessage> _notificationMsgs = new ArrayList<ClusterManagerMessage>();
    private ConnectionConcierge _heartbeatConnection = null;

    private final ExecutorService _executor;

    private ClusterServiceAdapter _currentServiceAdapter;

    @Inject
    private List<ClusterServiceAdapter> _serviceAdapters;

    @Inject
    private ManagementServerHostDao _mshostDao;
    @Inject
    private ManagementServerHostPeerDao _mshostPeerDao;

    protected Dispatcher _dispatcher;

    //
    // pay attention to _mshostId and _msid
    // _mshostId is the primary key of management host table
    // _msid is the unique persistent identifier that peer name is based upon
    //
    private Long _mshostId = null;
    protected long _msId = ManagementServerNode.getManagementServerId();
    protected long _runId = System.currentTimeMillis();

    private boolean _peerScanInited = false;

    private String _clusterNodeIP = "127.0.0.1";

    private final List<ClusterServicePdu> _clusterPduOutgoingQueue = new ArrayList<ClusterServicePdu>();
    private final List<ClusterServicePdu> _clusterPduIncomingQueue = new ArrayList<ClusterServicePdu>();
    private final Map<Long, ClusterServiceRequestPdu> _outgoingPdusWaitingForAck = new HashMap<Long, ClusterServiceRequestPdu>();

    public ClusterManagerImpl() {
        _clusterPeers = new HashMap<String, ClusterService>();

        // executor to perform remote-calls in another thread context, to avoid potential
        // recursive remote calls between nodes
        //
        _executor = Executors.newCachedThreadPool(new NamedThreadFactory("Cluster-Worker"));
        setRunLevel(ComponentLifecycle.RUN_LEVEL_FRAMEWORK);
    }

    private void registerRequestPdu(final ClusterServiceRequestPdu pdu) {
        synchronized (_outgoingPdusWaitingForAck) {
            _outgoingPdusWaitingForAck.put(pdu.getSequenceId(), pdu);
        }
    }

    @Override
    public void registerDispatcher(final Dispatcher dispatcher) {
        _dispatcher = dispatcher;
    }

    private ClusterServiceRequestPdu popRequestPdu(final long ackSequenceId) {
        synchronized (_outgoingPdusWaitingForAck) {
            if (_outgoingPdusWaitingForAck.get(ackSequenceId) != null) {
                final ClusterServiceRequestPdu pdu = _outgoingPdusWaitingForAck.get(ackSequenceId);
                _outgoingPdusWaitingForAck.remove(ackSequenceId);
                return pdu;
            }
        }

        return null;
    }

    private void cancelClusterRequestToPeer(final String strPeer) {
        final List<ClusterServiceRequestPdu> candidates = new ArrayList<ClusterServiceRequestPdu>();
        synchronized (_outgoingPdusWaitingForAck) {
            for (final Map.Entry<Long, ClusterServiceRequestPdu> entry : _outgoingPdusWaitingForAck.entrySet()) {
                if (entry.getValue().getDestPeer().equalsIgnoreCase(strPeer)) {
                    candidates.add(entry.getValue());
                }
            }

            for (final ClusterServiceRequestPdu pdu : candidates) {
                _outgoingPdusWaitingForAck.remove(pdu.getSequenceId());
            }
        }

        for (final ClusterServiceRequestPdu pdu : candidates) {
            s_logger.warn("Cancel cluster request PDU to peer: " + strPeer + ", pdu: " + pdu.getJsonPackage());
            synchronized (pdu) {
                pdu.notifyAll();
            }
        }
    }

    private void addOutgoingClusterPdu(final ClusterServicePdu pdu) {
        synchronized (_clusterPduOutgoingQueue) {
            _clusterPduOutgoingQueue.add(pdu);
            _clusterPduOutgoingQueue.notifyAll();
        }
    }

    private ClusterServicePdu popOutgoingClusterPdu(final long timeoutMs) {
        synchronized (_clusterPduOutgoingQueue) {
            try {
                _clusterPduOutgoingQueue.wait(timeoutMs);
            } catch (final InterruptedException e) {
            }

            if (_clusterPduOutgoingQueue.size() > 0) {
                final ClusterServicePdu pdu = _clusterPduOutgoingQueue.get(0);
                _clusterPduOutgoingQueue.remove(0);
                return pdu;
            }
        }
        return null;
    }

    private void addIncomingClusterPdu(final ClusterServicePdu pdu) {
        synchronized (_clusterPduIncomingQueue) {
            _clusterPduIncomingQueue.add(pdu);
            _clusterPduIncomingQueue.notifyAll();
        }
    }

    private ClusterServicePdu popIncomingClusterPdu(final long timeoutMs) {
        synchronized (_clusterPduIncomingQueue) {
            try {
                _clusterPduIncomingQueue.wait(timeoutMs);
            } catch (final InterruptedException e) {
            }

            if (_clusterPduIncomingQueue.size() > 0) {
                final ClusterServicePdu pdu = _clusterPduIncomingQueue.get(0);
                _clusterPduIncomingQueue.remove(0);
                return pdu;
            }
        }
        return null;
    }

    private Runnable getClusterPduSendingTask() {
        return new ManagedContextRunnable() {
            @Override
            protected void runInContext() {
                onSendingClusterPdu();
            }
        };
    }

    private Runnable getClusterPduNotificationTask() {
        return new ManagedContextRunnable() {
            @Override
            protected void runInContext() {
                onNotifyingClusterPdu();
            }
        };
    }

    private void onSendingClusterPdu() {
        while (true) {
            try {
                final ClusterServicePdu pdu = popOutgoingClusterPdu(1000);
                if (pdu == null) {
                    continue;
                }

                ClusterService peerService = null;
                for (int i = 0; i < 2; i++) {
                    try {
                        peerService = getPeerService(pdu.getDestPeer());
                    } catch (final RemoteException e) {
                        s_logger.error("Unable to get cluster service on peer : " + pdu.getDestPeer());
                    }

                    if (peerService != null) {
                        try {
                            if (s_logger.isDebugEnabled()) {
                                s_logger.debug("Cluster PDU " + getSelfPeerName() + " -> " + pdu.getDestPeer() + ". agent: " + pdu.getAgentId() + ", pdu seq: " +
                                        pdu.getSequenceId() + ", pdu ack seq: " + pdu.getAckSequenceId() + ", json: " + pdu.getJsonPackage());
                            }

                            final Profiler profiler = new Profiler();
                            profiler.start();

                            final String strResult = peerService.execute(pdu);
                            profiler.stop();

                            if (s_logger.isDebugEnabled()) {
                                s_logger.debug("Cluster PDU " + getSelfPeerName() + " -> " + pdu.getDestPeer() + " completed. time: " +
                                        profiler.getDurationInMillis() + "ms. agent: " + pdu.getAgentId() + ", pdu seq: " + pdu.getSequenceId() +
                                        ", pdu ack seq: " + pdu.getAckSequenceId() + ", json: " + pdu.getJsonPackage());
                            }

                            if ("true".equals(strResult)) {
                                break;
                            }

                        } catch (final RemoteException e) {
                            invalidatePeerService(pdu.getDestPeer());
                            if (s_logger.isInfoEnabled()) {
                                s_logger.info("Exception on remote execution, peer: " + pdu.getDestPeer() + ", iteration: " + i + ", exception message :" +
                                        e.getMessage());
                            }
                        }
                    }
                }
            } catch (final Throwable e) {
                s_logger.error("Unexcpeted exception: ", e);
            }
        }
    }

    private void onNotifyingClusterPdu() {
        while (true) {
            try {
                final ClusterServicePdu pdu = popIncomingClusterPdu(1000);
                if (pdu == null) {
                    continue;
                }

                _executor.execute(new ManagedContextRunnable() {
                    @Override
                    protected void runInContext() {
                        if (pdu.getPduType() == ClusterServicePdu.PDU_TYPE_RESPONSE) {
                            final ClusterServiceRequestPdu requestPdu = popRequestPdu(pdu.getAckSequenceId());
                            if (requestPdu != null) {
                                requestPdu.setResponseResult(pdu.getJsonPackage());
                                synchronized (requestPdu) {
                                    requestPdu.notifyAll();
                                }
                            } else {
                                s_logger.warn("Original request has already been cancelled. pdu: " + pdu.getJsonPackage());
                            }
                        } else {
                            String result = _dispatcher.dispatch(pdu);
                            if (result == null) {
                                result = "";
                            }

                            if (pdu.getPduType() == ClusterServicePdu.PDU_TYPE_REQUEST) {
                                final ClusterServicePdu responsePdu = new ClusterServicePdu();
                                responsePdu.setPduType(ClusterServicePdu.PDU_TYPE_RESPONSE);
                                responsePdu.setSourcePeer(pdu.getDestPeer());
                                responsePdu.setDestPeer(pdu.getSourcePeer());
                                responsePdu.setAckSequenceId(pdu.getSequenceId());
                                responsePdu.setJsonPackage(result);

                                addOutgoingClusterPdu(responsePdu);
                            }
                        }
                    }
                });
            } catch (final Throwable e) {
                s_logger.error("Unexcpeted exception: ", e);
            }
        }
    }

    @Override
    public void OnReceiveClusterServicePdu(final ClusterServicePdu pdu) {
        addIncomingClusterPdu(pdu);
    }

    /**
     * called by DatabaseUpgradeChecker to see if there are other peers running.
     *
     * @param notVersion
     *            If version is passed in, the peers CANNOT be running at this version. If version is null, return true if any
     *            peer is running regardless of version.
     * @return true if there are peers running and false if not.
     */
    public static final boolean arePeersRunning(final String notVersion) {
        return false; // TODO: Leaving this for Kelven to take care of.
    }

    @Override
    public void broadcast(final long agentId, final String cmds) {
        final Date cutTime = DateUtil.currentGMTTime();

        final List<ManagementServerHostVO> peers = _mshostDao.getActiveList(new Date(cutTime.getTime() - HeartbeatThreshold.value()));
        for (final ManagementServerHostVO peer : peers) {
            final String peerName = Long.toString(peer.getMsid());
            if (getSelfPeerName().equals(peerName)) {
                continue; // Skip myself.
            }
            try {
                if (s_logger.isDebugEnabled()) {
                    s_logger.debug("Forwarding " + cmds + " to " + peer.getMsid());
                }
                executeAsync(peerName, agentId, cmds, true);
            } catch (final Exception e) {
                s_logger.warn("Caught exception while talkign to " + peer.getMsid());
            }
        }
    }

    public void executeAsync(final String strPeer, final long agentId, final String cmds, final boolean stopOnError) {
        final ClusterServicePdu pdu = new ClusterServicePdu();
        pdu.setSourcePeer(getSelfPeerName());
        pdu.setDestPeer(strPeer);
        pdu.setAgentId(agentId);
        pdu.setJsonPackage(cmds);
        pdu.setStopOnError(true);
        addOutgoingClusterPdu(pdu);
    }

    @Override
    public String execute(final String strPeer, final long agentId, final String cmds, final boolean stopOnError) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug(getSelfPeerName() + " -> " + strPeer + "." + agentId + " " + cmds);
        }

        final ClusterServiceRequestPdu pdu = new ClusterServiceRequestPdu();
        pdu.setSourcePeer(getSelfPeerName());
        pdu.setDestPeer(strPeer);
        pdu.setAgentId(agentId);
        pdu.setJsonPackage(cmds);
        pdu.setStopOnError(stopOnError);
        registerRequestPdu(pdu);
        addOutgoingClusterPdu(pdu);

        synchronized (pdu) {
            try {
                pdu.wait();
            } catch (final InterruptedException e) {
            }
        }

        if (s_logger.isDebugEnabled()) {
            s_logger.debug(getSelfPeerName() + " -> " + strPeer + "." + agentId + " completed. result: " + pdu.getResponseResult());
        }

        if (pdu.getResponseResult() != null && pdu.getResponseResult().length() > 0) {
            return pdu.getResponseResult();
        }

        return null;
    }

    @Override
    public ManagementServerHostVO getPeer(final String mgmtServerId) {
        return _mshostDao.findByMsid(Long.parseLong(mgmtServerId));
    }

    @Override
    public String getSelfPeerName() {
        return Long.toString(_msId);
    }

    public String getSelfNodeIP() {
        return _clusterNodeIP;
    }

    @Override
    public void registerListener(final ClusterManagerListener listener) {
        // Note : we don't check duplicates
        synchronized (_listeners) {

            s_logger.info("register cluster listener " + listener.getClass());

            _listeners.add(listener);
        }
    }

    @Override
    public void unregisterListener(final ClusterManagerListener listener) {
        synchronized (_listeners) {
            s_logger.info("unregister cluster listener " + listener.getClass());

            _listeners.remove(listener);
        }
    }

    public void notifyNodeJoined(final List<ManagementServerHostVO> nodeList) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("Notify management server node join to listeners.");

            for (final ManagementServerHostVO mshost : nodeList) {
                s_logger.debug("Joining node, IP: " + mshost.getServiceIP() + ", msid: " + mshost.getMsid());
            }
        }

        synchronized (_listeners) {
            for (final ClusterManagerListener listener : _listeners) {
                listener.onManagementNodeJoined(nodeList, _mshostId);
            }
        }

        SubscriptionMgr.getInstance().notifySubscribers(ClusterManager.ALERT_SUBJECT, this, new ClusterNodeJoinEventArgs(_mshostId, nodeList));
    }

    public void notifyNodeLeft(final List<ManagementServerHostVO> nodeList) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("Notify management server node left to listeners.");
        }

        for (final ManagementServerHostVO mshost : nodeList) {
            if (s_logger.isDebugEnabled()) {
                s_logger.debug("Leaving node, IP: " + mshost.getServiceIP() + ", msid: " + mshost.getMsid());
            }
            cancelClusterRequestToPeer(String.valueOf(mshost.getMsid()));
        }

        synchronized (_listeners) {
            for (final ClusterManagerListener listener : _listeners) {
                listener.onManagementNodeLeft(nodeList, _mshostId);
            }
        }

        SubscriptionMgr.getInstance().notifySubscribers(ClusterManager.ALERT_SUBJECT, this, new ClusterNodeLeftEventArgs(_mshostId, nodeList));
    }

    public void notifyNodeIsolated() {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("Notify management server node isolation to listeners");
        }

        synchronized (_listeners) {
            for (final ClusterManagerListener listener : _listeners) {
                listener.onManagementNodeIsolated();
            }
        }
    }

    public ClusterService getPeerService(final String strPeer) throws RemoteException {
        synchronized (_clusterPeers) {
            if (_clusterPeers.containsKey(strPeer)) {
                return _clusterPeers.get(strPeer);
            }
        }

        final ClusterService service = _currentServiceAdapter.getPeerService(strPeer);

        if (service != null) {
            synchronized (_clusterPeers) {
                // re-check the peer map again to deal with the
                // race conditions
                if (!_clusterPeers.containsKey(strPeer)) {
                    _clusterPeers.put(strPeer, service);
                }
            }
        }

        return service;
    }

    public void invalidatePeerService(final String strPeer) {
        synchronized (_clusterPeers) {
            if (_clusterPeers.containsKey(strPeer)) {
                _clusterPeers.remove(strPeer);
            }
        }
    }

    private Runnable getHeartbeatTask() {
        return new ManagedContextRunnable() {
            @Override
            protected void runInContext() {
                final TransactionLegacy txn = TransactionLegacy.open("ClusterHeartbeat");
                try {
                    final Profiler profiler = new Profiler();
                    final Profiler profilerHeartbeatUpdate = new Profiler();
                    final Profiler profilerPeerScan = new Profiler();

                    try {
                        profiler.start();

                        profilerHeartbeatUpdate.start();
                        txn.transitToUserManagedConnection(getHeartbeatConnection());
                        if (s_logger.isTraceEnabled()) {
                            s_logger.trace("Cluster manager heartbeat update, id:" + _mshostId);
                        }

                        _mshostDao.update(_mshostId, _runId, DateUtil.currentGMTTime());
                        profilerHeartbeatUpdate.stop();

                        profilerPeerScan.start();
                        if (s_logger.isTraceEnabled()) {
                            s_logger.trace("Cluster manager peer-scan, id:" + _mshostId);
                        }

                        if (!_peerScanInited) {
                            _peerScanInited = true;
                            initPeerScan();
                        }

                        peerScan();
                        profilerPeerScan.stop();

                    } finally {
                        profiler.stop();

                        if (profiler.getDurationInMillis() >= HeartbeatInterval.value()) {
                            if (s_logger.isDebugEnabled()) {
                                s_logger.debug("Management server heartbeat takes too long to finish. profiler: " + profiler.toString() + ", profilerHeartbeatUpdate: " +
                                        profilerHeartbeatUpdate.toString() + ", profilerPeerScan: " + profilerPeerScan.toString());
                            }
                        }
                    }

                } catch (final CloudRuntimeException e) {
                    s_logger.error("Runtime DB exception ", e.getCause());

                    if (e.getCause() instanceof ClusterInvalidSessionException) {
                        s_logger.error("Invalid cluster session found, fence it");
                        queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeIsolated));
                    }

                    if (isRootCauseConnectionRelated(e.getCause())) {
                        invalidHeartbeatConnection();
                    }
                } catch (final ActiveFencingException e) {
                    queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeIsolated));
                } catch (final Throwable e) {
                    s_logger.error("Unexpected exception in cluster heartbeat", e);
                    if (isRootCauseConnectionRelated(e.getCause())) {
                        invalidHeartbeatConnection();
                    }
                } finally {
                    txn.transitToAutoManagedConnection(TransactionLegacy.CLOUD_DB);
                    txn.close("ClusterHeartbeat");
                }
            }
        };
    }

    private boolean isRootCauseConnectionRelated(Throwable e) {
        while (e != null) {
            if (e instanceof SQLRecoverableException || e instanceof SQLNonTransientException) {
                return true;
            }

            e = e.getCause();
        }

        return false;
    }

    private Connection getHeartbeatConnection() throws SQLException {
        if (_heartbeatConnection == null) {
            final Connection conn = TransactionLegacy.getStandaloneConnectionWithException();
            _heartbeatConnection = new ConnectionConcierge("ClusterManagerHeartbeat", conn, false);
        }

        return _heartbeatConnection.conn();
    }

    private void invalidHeartbeatConnection() {
        if (_heartbeatConnection != null) {
            final Connection conn = TransactionLegacy.getStandaloneConnection();
            if (conn != null) {
                _heartbeatConnection.reset(conn);
            } else {
                s_logger.error("DB communication problem detected, fence it");
                queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeIsolated));
            }
            // The stand-alone connection does not have to be closed here because there will be another reference to it.
            // As a matter of fact, it will be assigned to the connection instance variable in the ConnectionConcierge class.
        }
    }

    private Runnable getNotificationTask() {
        return new ManagedContextRunnable() {
            @Override
            protected void runInContext() {
                while (true) {
                    synchronized (_notificationMsgs) {
                        try {
                            _notificationMsgs.wait(1000);
                        } catch (final InterruptedException e) {
                        }
                    }

                    ClusterManagerMessage msg = null;
                    while ((msg = getNextNotificationMessage()) != null) {
                        try {
                            switch (msg.getMessageType()) {
                            case nodeAdded:
                                if (msg.getNodes() != null && msg.getNodes().size() > 0) {
                                    final Profiler profiler = new Profiler();
                                    profiler.start();

                                    notifyNodeJoined(msg.getNodes());

                                    profiler.stop();
                                    if (profiler.getDurationInMillis() > 1000) {
                                        if (s_logger.isDebugEnabled()) {
                                            s_logger.debug("Notifying management server join event took " + profiler.getDurationInMillis() + " ms");
                                        }
                                    } else {
                                        s_logger.warn("Notifying management server join event took " + profiler.getDurationInMillis() + " ms");
                                    }
                                }
                                break;

                            case nodeRemoved:
                                if (msg.getNodes() != null && msg.getNodes().size() > 0) {
                                    final Profiler profiler = new Profiler();
                                    profiler.start();

                                    notifyNodeLeft(msg.getNodes());

                                    profiler.stop();
                                    if (profiler.getDurationInMillis() > 1000) {
                                        if (s_logger.isDebugEnabled()) {
                                            s_logger.debug("Notifying management server leave event took " + profiler.getDurationInMillis() + " ms");
                                        }
                                    } else {
                                        s_logger.warn("Notifying management server leave event took " + profiler.getDurationInMillis() + " ms");
                                    }
                                }
                                break;

                            case nodeIsolated:
                                notifyNodeIsolated();
                                break;

                            default:
                                assert false;
                                break;
                            }

                        } catch (final Throwable e) {
                            s_logger.warn("Unexpected exception during cluster notification. ", e);
                        }
                    }

                    try {
                        Thread.sleep(1000);
                    } catch (final InterruptedException e) {
                    }
                }
            }
        };
    }

    private void queueNotification(final ClusterManagerMessage msg) {
        synchronized (_notificationMsgs) {
            _notificationMsgs.add(msg);
            _notificationMsgs.notifyAll();
        }

        switch (msg.getMessageType()) {
        case nodeAdded: {
            final List<ManagementServerHostVO> l = msg.getNodes();
            if (l != null && l.size() > 0) {
                for (final ManagementServerHostVO mshost : l) {
                    _mshostPeerDao.updatePeerInfo(_mshostId, mshost.getId(), mshost.getRunid(), ManagementServerHost.State.Up);
                }
            }
        }
        break;

        case nodeRemoved: {
            final List<ManagementServerHostVO> l = msg.getNodes();
            if (l != null && l.size() > 0) {
                for (final ManagementServerHostVO mshost : l) {
                    _mshostPeerDao.updatePeerInfo(_mshostId, mshost.getId(), mshost.getRunid(), ManagementServerHost.State.Down);
                }
            }
        }
        break;

        default:
            break;
        }
    }

    private ClusterManagerMessage getNextNotificationMessage() {
        synchronized (_notificationMsgs) {
            if (_notificationMsgs.size() > 0) {
                return _notificationMsgs.remove(0);
            }
        }

        return null;
    }

    private void initPeerScan() {
        // upon startup, for all inactive management server nodes that we see at startup time, we will send notification also to help upper layer perform
        // missed cleanup
        final Date cutTime = DateUtil.currentGMTTime();
        final List<ManagementServerHostVO> inactiveList = _mshostDao.getInactiveList(new Date(cutTime.getTime() - HeartbeatThreshold.value()));

        // We don't have foreign key constraints to enforce the mgmt_server_id integrity in host table, when user manually
        // remove records from mshost table, this will leave orphan mgmt_serve_id reference in host table.
        final List<Long> orphanList = _mshostDao.listOrphanMsids();
        if (orphanList.size() > 0) {
            for (final Long orphanMsid : orphanList) {
                // construct fake ManagementServerHostVO based on orphan MSID
                s_logger.info("Add orphan management server msid found in host table to initial clustering notification, orphan msid: " + orphanMsid);
                inactiveList.add(new ManagementServerHostVO(orphanMsid, 0, "orphan", 0, new Date()));
            }
        } else {
            s_logger.info("We are good, no orphan management server msid in host table is found");
        }

        if (inactiveList.size() > 0) {
            if (s_logger.isInfoEnabled()) {
                s_logger.info("Found " + inactiveList.size() + " inactive management server node based on timestamp");
                for (final ManagementServerHostVO host : inactiveList) {
                    s_logger.info("management server node msid: " + host.getMsid() + ", name: " + host.getName() + ", service ip: " + host.getServiceIP() +
                            ", version: " + host.getVersion());
                }
            }

            final List<ManagementServerHostVO> downHostList = new ArrayList<ManagementServerHostVO>();
            for (final ManagementServerHostVO host : inactiveList) {
                if (!pingManagementNode(host)) {
                    s_logger.warn("Management node " + host.getId() + " is detected inactive by timestamp and also not pingable");
                    downHostList.add(host);
                }
            }

            if (downHostList.size() > 0) {
                queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeRemoved, downHostList));
            }
        } else {
            s_logger.info("No inactive management server node found");
        }
    }

    private void peerScan() throws ActiveFencingException {
        final Date cutTime = DateUtil.currentGMTTime();

        final Profiler profiler = new Profiler();
        profiler.start();

        final Profiler profilerQueryActiveList = new Profiler();
        profilerQueryActiveList.start();
        final List<ManagementServerHostVO> currentList = _mshostDao.getActiveList(new Date(cutTime.getTime() - HeartbeatThreshold.value()));
        profilerQueryActiveList.stop();

        final Profiler profilerSyncClusterInfo = new Profiler();
        profilerSyncClusterInfo.start();
        final List<ManagementServerHostVO> removedNodeList = new ArrayList<ManagementServerHostVO>();
        final List<ManagementServerHostVO> invalidatedNodeList = new ArrayList<ManagementServerHostVO>();

        if (_mshostId != null) {

            if (_mshostPeerDao.countStateSeenInPeers(_mshostId, _runId, ManagementServerHost.State.Down) > 0) {
                final String msg =
                        "We have detected that at least one management server peer reports that this management server is down, perform active fencing to avoid split-brain situation";
                s_logger.error(msg);
                throw new ActiveFencingException(msg);
            }

            // only if we have already attached to cluster, will we start to check leaving nodes
            for (final Map.Entry<Long, ManagementServerHostVO> entry : _activePeers.entrySet()) {

                final ManagementServerHostVO current = getInListById(entry.getKey(), currentList);
                if (current == null) {
                    if (entry.getKey().longValue() != _mshostId.longValue()) {
                        if (s_logger.isDebugEnabled()) {
                            s_logger.debug("Detected management node left, id:" + entry.getKey() + ", nodeIP:" + entry.getValue().getServiceIP());
                        }
                        removedNodeList.add(entry.getValue());
                    }
                } else {
                    if (current.getRunid() == 0) {
                        if (entry.getKey().longValue() != _mshostId.longValue()) {
                            if (s_logger.isDebugEnabled()) {
                                s_logger.debug("Detected management node left because of invalidated session, id:" + entry.getKey() + ", nodeIP:" +
                                        entry.getValue().getServiceIP());
                            }
                            invalidatedNodeList.add(entry.getValue());
                        }
                    } else {
                        if (entry.getValue().getRunid() != current.getRunid()) {
                            if (s_logger.isDebugEnabled()) {
                                s_logger.debug("Detected management node left and rejoined quickly, id:" + entry.getKey() + ", nodeIP:" + entry.getValue().getServiceIP());
                            }

                            entry.getValue().setRunid(current.getRunid());
                        }
                    }
                }
            }
        }
        profilerSyncClusterInfo.stop();

        final Profiler profilerInvalidatedNodeList = new Profiler();
        profilerInvalidatedNodeList.start();
        // process invalidated node list
        if (invalidatedNodeList.size() > 0) {
            for (final ManagementServerHostVO mshost : invalidatedNodeList) {
                _activePeers.remove(mshost.getId());
                try {
                    JmxUtil.unregisterMBean("ClusterManager", "Node " + mshost.getId());
                } catch (final Exception e) {
                    s_logger.warn("Unable to deregiester cluster node from JMX monitoring due to exception " + e.toString());
                }
            }

            queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeRemoved, invalidatedNodeList));
        }
        profilerInvalidatedNodeList.stop();

        final Profiler profilerRemovedList = new Profiler();
        profilerRemovedList.start();
        // process removed node list
        final Iterator<ManagementServerHostVO> it = removedNodeList.iterator();
        while (it.hasNext()) {
            final ManagementServerHostVO mshost = it.next();
            if (!pingManagementNode(mshost)) {
                s_logger.warn("Management node " + mshost.getId() + " is detected inactive by timestamp and also not pingable");
                _activePeers.remove(mshost.getId());
                try {
                    JmxUtil.unregisterMBean("ClusterManager", "Node " + mshost.getId());
                } catch (final Exception e) {
                    s_logger.warn("Unable to deregiester cluster node from JMX monitoring due to exception " + e.toString());
                }
            } else {
                s_logger.info("Management node " + mshost.getId() + " is detected inactive by timestamp but is pingable");
                it.remove();
            }
        }

        if (removedNodeList.size() > 0) {
            queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeRemoved, removedNodeList));
        }
        profilerRemovedList.stop();

        final List<ManagementServerHostVO> newNodeList = new ArrayList<ManagementServerHostVO>();
        for (final ManagementServerHostVO mshost : currentList) {
            if (!_activePeers.containsKey(mshost.getId())) {
                _activePeers.put(mshost.getId(), mshost);

                if (s_logger.isDebugEnabled()) {
                    s_logger.debug("Detected management node joined, id:" + mshost.getId() + ", nodeIP:" + mshost.getServiceIP());
                }
                newNodeList.add(mshost);

                try {
                    JmxUtil.registerMBean("ClusterManager", "Node " + mshost.getId(), new ClusterManagerMBeanImpl(this, mshost));
                } catch (final Exception e) {
                    s_logger.warn("Unable to regiester cluster node into JMX monitoring due to exception " + ExceptionUtil.toString(e));
                }
            }
        }

        if (newNodeList.size() > 0) {
            queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeAdded, newNodeList));
        }

        profiler.stop();

        if (profiler.getDurationInMillis() >= HeartbeatInterval.value()) {
            if (s_logger.isDebugEnabled()) {
                s_logger.debug("Peer scan takes too long to finish. profiler: " + profiler.toString() + ", profilerQueryActiveList: " +
                        profilerQueryActiveList.toString() + ", profilerSyncClusterInfo: " + profilerSyncClusterInfo.toString() + ", profilerInvalidatedNodeList: " +
                        profilerInvalidatedNodeList.toString() + ", profilerRemovedList: " + profilerRemovedList.toString());
            }
        }
    }

    private static ManagementServerHostVO getInListById(final Long id, final List<ManagementServerHostVO> l) {
        for (final ManagementServerHostVO mshost : l) {
            if (mshost.getId() == id) {
                return mshost;
            }
        }
        return null;
    }

    @Override
    @DB
    public boolean start() {
        if (s_logger.isInfoEnabled()) {
            s_logger.info("Starting Cluster manager, msid : " + _msId);
        }

        final ManagementServerHostVO mshost = Transaction.execute(new TransactionCallback<ManagementServerHostVO>() {
            @Override
            public ManagementServerHostVO doInTransaction(final TransactionStatus status) {

                final Class<?> c = this.getClass();
                final String version = c.getPackage().getImplementationVersion();

                ManagementServerHostVO mshost = _mshostDao.findByMsid(_msId);
                if (mshost == null) {
                    mshost = new ManagementServerHostVO();
                    mshost.setMsid(_msId);
                    mshost.setRunid(_runId);
                    mshost.setName(NetUtils.getHostName());
                    mshost.setVersion(version);
                    mshost.setServiceIP(_clusterNodeIP);
                    mshost.setServicePort(_currentServiceAdapter.getServicePort());
                    mshost.setLastUpdateTime(DateUtil.currentGMTTime());
                    mshost.setRemoved(null);
                    mshost.setAlertCount(0);
                    mshost.setState(ManagementServerHost.State.Up);
                    _mshostDao.persist(mshost);
                    if (s_logger.isInfoEnabled()) {
                        s_logger.info("New instance of management server msid " + _msId + ", runId " + _runId + " is being started");
                    }
                } else {
                    _mshostDao.update(mshost.getId(), _runId, NetUtils.getHostName(), version, _clusterNodeIP, _currentServiceAdapter.getServicePort(),
                            DateUtil.currentGMTTime());
                    if (s_logger.isInfoEnabled()) {
                        s_logger.info("Management server " + _msId + ", runId " + _runId + " is being started");
                    }
                }

                return mshost;
            }
        });

        _mshostId = mshost.getId();
        if (s_logger.isInfoEnabled()) {
            s_logger.info("Management server (host id : " + _mshostId + ") is being started at " + _clusterNodeIP + ":" + _currentServiceAdapter.getServicePort());
        }

        _mshostPeerDao.clearPeerInfo(_mshostId);

        // use seperate thread for heartbeat updates
        _heartbeatScheduler.scheduleAtFixedRate(getHeartbeatTask(), HeartbeatInterval.value(), HeartbeatInterval.value(), TimeUnit.MILLISECONDS);
        _notificationExecutor.submit(getNotificationTask());

        if (s_logger.isInfoEnabled()) {
            s_logger.info("Cluster manager was started successfully");
        }

        return true;
    }

    @Override
    @DB
    public boolean stop() {
        if (s_logger.isInfoEnabled()) {
            s_logger.info("Stopping Cluster manager, msid : " + _msId);
        }

        if (_mshostId != null) {
            final ManagementServerHostVO mshost = _mshostDao.findByMsid(_msId);
            mshost.setState(ManagementServerHost.State.Down);
            _mshostDao.update(_mshostId, mshost);
        }

        _heartbeatScheduler.shutdownNow();
        _executor.shutdownNow();

        try {
            _heartbeatScheduler.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT, TimeUnit.MILLISECONDS);
            _executor.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT, TimeUnit.MILLISECONDS);
        } catch (final InterruptedException e) {
        }

        if (s_logger.isInfoEnabled()) {
            s_logger.info("Cluster manager is stopped");
        }

        return true;
    }

    @Override
    public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
        if (s_logger.isInfoEnabled()) {
            s_logger.info("Start configuring cluster manager : " + name);
        }

        final Properties dbProps = DbProperties.getDbProperties();
        _clusterNodeIP = dbProps.getProperty("cluster.node.IP");
        if (_clusterNodeIP == null) {
            _clusterNodeIP = "127.0.0.1";
        }
        _clusterNodeIP = _clusterNodeIP.trim();

        if (s_logger.isInfoEnabled()) {
            s_logger.info("Cluster node IP : " + _clusterNodeIP);
        }

        if (!NetUtils.isLocalAddress(_clusterNodeIP)) {
            throw new ConfigurationException("cluster node IP should be valid local address where the server is running, please check your configuration");
        }

        for (int i = 0; i < DEFAULT_OUTGOING_WORKERS; i++) {
            _executor.execute(getClusterPduSendingTask());
        }

        // notification task itself in turn works as a task dispatcher
        _executor.execute(getClusterPduNotificationTask());

        if (_serviceAdapters == null) {
            throw new ConfigurationException("Unable to get cluster service adapters");
        }
        _currentServiceAdapter = _serviceAdapters.get(0);

        if (_currentServiceAdapter == null) {
            throw new ConfigurationException("Unable to set current cluster service adapter");
        }

        checkConflicts();

        if (s_logger.isInfoEnabled()) {
            s_logger.info("Cluster manager is configured.");
        }
        return true;
    }

    @Override
    public long getManagementNodeId() {
        return _msId;
    }

    @Override
    public long getCurrentRunId() {
        return _runId;
    }

    @Override
    public long getManagementRunId(final long msId) {
        final ManagementServerHostVO mshost = _mshostDao.findByMsid(msId);
        if (mshost != null) {
            return mshost.getRunid();
        }
        return -1;
    }

    public boolean isManagementNodeAlive(final long msid) {
        final ManagementServerHostVO mshost = _mshostDao.findByMsid(msid);
        if (mshost != null) {
            if (mshost.getLastUpdateTime().getTime() >= DateUtil.currentGMTTime().getTime() - HeartbeatThreshold.value()) {
                return true;
            }
        }

        return false;
    }

    public boolean pingManagementNode(final long msid) {
        final ManagementServerHostVO mshost = _mshostDao.findByMsid(msid);
        if (mshost == null) {
            return false;
        }

        return pingManagementNode(mshost);
    }

    @Override
    public String getConfigComponentName() {
        return ClusterManager.class.getSimpleName();
    }

    @Override
    public ConfigKey<?>[] getConfigKeys() {
        return new ConfigKey<?>[] {HeartbeatInterval, HeartbeatThreshold};
    }

    private boolean pingManagementNode(final ManagementServerHostVO mshost) {

        final String targetIp = mshost.getServiceIP();
        if ("127.0.0.1".equals(targetIp) || "0.0.0.0".equals(targetIp)) {
            s_logger.info("ping management node cluster service can not be performed on self");
            return false;
        }

        int retry = 10;
        while (--retry > 0) {
            SocketChannel sch = null;
            try {
                s_logger.info("Trying to connect to " + targetIp);
                sch = SocketChannel.open();
                sch.configureBlocking(true);
                sch.socket().setSoTimeout(5000);

                final InetSocketAddress addr = new InetSocketAddress(targetIp, mshost.getServicePort());
                sch.connect(addr);
                return true;
            } catch (final IOException e) {
                if (e instanceof ConnectException) {
                    s_logger.error("Unable to ping management server at " + targetIp + ":" + mshost.getServicePort() + " due to ConnectException", e);
                    return false;
                }
            } finally {
                if (sch != null) {
                    try {
                        sch.close();
                    } catch (final IOException e) {
                    }
                }
            }

            try {
                Thread.sleep(1000);
            } catch (final InterruptedException ex) {
            }
        }

        s_logger.error("Unable to ping management server at " + targetIp + ":" + mshost.getServicePort() + " after retries");
        return false;
    }

    public int getHeartbeatInterval() {
        return HeartbeatInterval.value();
    }

    private void checkConflicts() throws ConfigurationException {
        final Date cutTime = DateUtil.currentGMTTime();
        final List<ManagementServerHostVO> peers = _mshostDao.getActiveList(new Date(cutTime.getTime() - HeartbeatThreshold.value()));
        for (final ManagementServerHostVO peer : peers) {
            final String peerIP = peer.getServiceIP().trim();
            if (_clusterNodeIP.equals(peerIP)) {
                if ("127.0.0.1".equals(_clusterNodeIP)) {
                    if (pingManagementNode(peer.getMsid())) {
                        final String msg = "Detected another management node with localhost IP is already running, please check your cluster configuration";
                        s_logger.error(msg);
                        throw new ConfigurationException(msg);
                    } else {
                        final String msg =
                                "Detected another management node with localhost IP is considered as running in DB, however it is not pingable, we will continue cluster initialization with this management server node";
                        s_logger.info(msg);
                    }
                } else {
                    if (pingManagementNode(peer.getMsid())) {
                        final String msg =
                                "Detected that another management node with the same IP " + peer.getServiceIP() +
                                " is already running, please check your cluster configuration";
                        s_logger.error(msg);
                        throw new ConfigurationException(msg);
                    } else {
                        final String msg =
                                "Detected that another management node with the same IP " + peer.getServiceIP() +
                                " is considered as running in DB, however it is not pingable, we will continue cluster initialization with this management server node";
                        s_logger.info(msg);
                    }
                }
            }
        }
    }

}
