#!/bin/bash
#
# @@@ START COPYRIGHT @@@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# @@@ END COPYRIGHT @@@
#
# Execute this test script from $TRAF_HOME/monitor/test.
#
# Cluster environment setup:
#    - create directories
#      cd $TRAF_HOME
#      edb_pdsh -a mkdir $PWD/monitor
#      edb_pdsh -a mkdir $PWD/monitor/test
#    - copy test files to $TRAF_HOME/monitor/test in all nodes
#      cd $TRAF_HOME/monitor/test
#      pdcp $(trafconf -wname) -p -r /home/sqft2/monitor/test/* $PWD/
#    - execute runtest -cluster
#      cd $TRAF_HOME/monitor/test
#      runtest -cluster
#
# Virtual environment setup:
#    - execute runtest -virtual
#      cd $TRAF_HOME/monitor/test
#      runtest -virtual
#

function Usage
{
    echo
    echo "runtest { -cluster | -virtual } [ -nogen | -trace | -test <num> ]"
    echo
    echo "Where: <num> is one of the following tests:"
    echo "         1     - Child Exit"
    echo "         2     - Multi-Node"
    echo "         3     - Registry"
    echo "         4     - Death Notice"
    echo "         5     - Persistent Process"
    echo "         6     - DTM Process"
    echo "         7     - Process Create"
    echo "         8     - Node down before startup"
    echo
}

function Cleanup
{
    monpkillall
    sqipcrm -local
}

if [ $# -lt 1 ]; then
    Usage
    exit 1
fi

cluster=
conf=
conf_persist=
virtual=
trace=
nogen=

declare -i test

let test=-1

if [ "$1" = "-cluster" ]; then
    cluster=-cluster
    conf="sqconfig.monitor.cluster"
    conf_persist="sqconfig.persist"
    shift
fi
if [ "$1" = "-virtual" ]; then
    virtual=-virtual
    conf="sqconfig.monitor.virtual"
    conf_persist="sqconfig.persist"
    shift
fi
if [ "$1" = "-nogen" ]; then
    nogen=${1}
    shift
fi
if [ "$1" = "-trace" ]; then
    trace=-t
    shift
fi
if [ "$1" = "-test" ]; then
    shift
    test=${1}
    shift
    if ( [ $test '<' 1 ] || [ $test '>' 8 ] ); then
        echo
        echo "Invalid test number (-test ${test})"
        Usage
        exit 1
    fi
fi

#
# Setup test execution
#
ARCH=`arch`
export PATH=$PATH:$PWD/Linux-${ARCH}/64/dbg
export TRAF_CLUSTER_ID=1
export TRAF_INSTANCE_ID=1
cd $TRAF_HOME/monitor/test
echo $PWD

if [ "$cluster" = "-cluster" ]; then
    echo ************ Executing test in cluster environment ************
else if [ "$virtual" = "-virtual" ]; then
    echo ************ Executing test in virtual environment ************
else
    Usage
    exit 1
fi
fi

if [ "$cluster" = "-cluster" ]; then
    edb_pdsh -a "rm -f $MPI_TMPDIR/monitor.port.*"
    edb_pdsh -a "rm $PWD/core* $PWD/*.lst $PWD/stdout_*"
else
    rm -f $MPI_TMPDIR/monitor.port.*
    rm core* *.lst stdout_*
    # Establish SQ virtual cluster parameters
    export SQ_VIRTUAL_NODES=6
    export SQ_VIRTUAL_NID=0
fi

if [ ! -z "$SQ_DTM_PERSISTENT_PROCESS" ]; then
    let dtm_persistent_process=`echo $SQ_DTM_PERSISTENT_PROCESS`
else
    let dtm_persistent_process="0"
fi

echo ************ Executing test $test ************

export SQ_SHELL_NOCWD=1
# Uncomment to enable LIO tracing in test programs
#export SQ_LOCAL_IO_SHELL_TRACE=1

genConf_Success=0
genConf_Failure=255

function Genconf() 
{
#
# Setup up monitor configuration
#
    [ $# -eq 1 ] || return $genConf_Failure

    case $1 in
    *[!0-9]*|"") 
               return $genConf_Failure ;;

    [1-5,7-8]) 
             conf_persist=sqconfig.persist  ;;
             
    [6])      
       conf_persist=sqconfig.persist.dtm  ;;
      
    esac

    echo "**********************************************************"
    echo ./montestgen $TRAF_HOME/monitor/test/$conf $TRAF_HOME/monitor/test/$conf_persist
    echo "**********************************************************"
    #./montestgen $TRAF_HOME/monitor/test/$conf $TRAF_HOME/monitor/test/$conf_persist
    
    ./montestgen $TRAF_HOME/monitor/test/$conf $TRAF_HOME/monitor/test/$conf_persist
    return $genConf_Success 
}

function Shutdown_status() 
{
#
# Verify if shutdown stopped the required processes
#
    ret=$(monpstat -h|wc -l)
    if [ $ret == "0" ] ; then
        echo "${*} Shutdown: PASSED"
    else
        echo "${*} Shutdown: FAILED"
    fi
}


#
# Execute tests
#
if ( [ $test '==' -1 ] || [ $test '==' 1 ] ); then
Cleanup
Genconf 1
[ $? -eq 0 ] || return 101

TestTitle="Child Exit Test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell <<eof
 startup
 delay 3
 ps
 exec {name \$CTRLR, nid 0, out $TRAF_HOME/monitor/test/childExit.lst} childExitCtrl $trace
 delay 3
 persist info PP 
 persist kill PP
 persist info ABC 
 persist kill ABC
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

 sleep 5
 monpstat
 Shutdown_status ${TestTitle} >> $PWD/childExit.lst
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     edb_pdsh -a grep PASSED $PWD/childExit.lst
     edb_pdsh -a grep FAILED $PWD/childExit.lst
 else
     grep PASSED $PWD/childExit.lst
     grep FAILED $PWD/childExit.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 2 ] ); then
Cleanup
Genconf 2
[ $? -eq 0 ] || return 101

TestTitle="Multi-Node test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell <<eof
 startup
 delay 3
 ! General tests for monitor functionality. This includes
 !  Open, Close, NewProcess, ProcessInfo and sync node state
 !  between nodes.
 node
 ! To enable tracing add "-t" argument to the end of the "exec" line
 exec {nowait,name \$SERV0,nid 0} server $trace
 exec {nowait,pri 5,name \$SERV1} server $trace
 ps
 exec {pri 10,name \$CLIENT,nid 0, out $TRAF_HOME/monitor/test/multiNode.lst} client $trace
 delay 3
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

 sleep 5
 monpstat
 Shutdown_status ${TestTitle} >> $PWD/multiNode.lst
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     edb_pdsh -a grep PASSED $PWD/multiNode.lst
     edb_pdsh -a grep FAILED $PWD/multiNode.lst
 else
     grep PASSED $PWD/multiNode.lst
     grep FAILED $PWD/multiNode.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 3 ] ); then
Cleanup
Genconf 3 
[ $? -eq 0 ] || return 101

TestTitle="Registry Test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell <<eof
 startup
 delay 3
 exec {name \$CTRLR, nid 0, out $TRAF_HOME/monitor/test/regTest.lst} regTestCtrl $trace
 delay 3
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

 sleep 5
 monpstat
 Shutdown_status ${TestTitle} >> $PWD/regTest.lst
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     edb_pdsh -a grep PASSED $PWD/regTest.lst
     edb_pdsh -a grep FAILED $PWD/regTest.lst
 else
     grep PASSED $PWD/regTest.lst
     grep FAILED $PWD/regTest.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 4 ] ); then
Cleanup
Genconf 4
[ $? -eq 0 ] || return 101

TestTitle="Death Notice Test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell <<eof
 startup
 delay 3
 ! Test to register and cancel process death
 ! notices from non-parent process.
 !
 exec {nowait,name \$serv0, out serv0.lst} server waitforclose $trace
 exec {nowait,name \$serv1, out serv1.lst} server waitforclose $trace
 exec {nowait,nid 0,name \$TM00,out tm00.lst} server waitforclose $trace
 exec {nowait,nid 1,name \$TM01,out tm01.lst} server waitforclose $trace
 exec {nowait,nid 2,name \$TM02,out tm02.lst} server waitforclose $trace
 ps
 ! To enable tracing add "-t" argument to the end of the "exec" line
 !
 exec {name \$DEATH, nid 0, out $TRAF_HOME/monitor/test/deathNotice.lst} deathNotice $trace
 delay 3
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

 sleep 5
 monpstat
 Shutdown_status ${TestTitle} >> $PWD/deathNotice.lst
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     edb_pdsh -a grep PASSED $PWD/deathNotice.lst
     edb_pdsh -a grep FAILED $PWD/deathNotice.lst
 else
     grep PASSED $PWD/deathNotice.lst
     grep FAILED $PWD/deathNotice.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 5 ] ); then
 if [ "$cluster" = "-cluster" ]; then
Cleanup
Genconf 5
[ $? -eq 0 ] || return 101

TestTitle="Persistent Process CLUSTER Test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell <<eof
 startup
 delay 3
 down 2 !
 delay 15
 exec {name \$PPROC, nid 0, out $TRAF_HOME/monitor/test/persistentProc.lst} persistentProc $trace
 delay 3
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

  sleep 5
  monpstat
  Shutdown_status ${TestTitle} >> $PWD/persistentProc.lst
  cstat -h
  edb_pdsh -a grep PASSED $PWD/persistentProc.lst
  edb_pdsh -a grep FAILED $PWD/persistentProc.lst

 else

Cleanup
Genconf 5
TestTitle="Persistent Process Test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell <<eof
 startup
 delay 3
 exec {name \$PPROC, nid 0, out $TRAF_HOME/monitor/test/persistentProc.lst} persistentProc $trace
 delay 3
 !shutdown
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

  sleep 5
  monpstat
  Shutdown_status ${TestTitle} >> $PWD/persistentProc.lst
  grep PASSED $PWD/persistentProc.lst
  grep FAILED $PWD/persistentProc.lst
 fi
fi

if ( [[ $dtm_persistent_process == "1" ]] &&
      ([ $test '==' -1 ] || [ $test '==' 6 ]) ); then
Cleanup
Genconf 6
[ $? -eq 0 ] || return 101

TestTitle="DTM Process Test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell <<eof
 startup
 delay 3
 persist info dtm
 exec {name \$DTMCTRL, nid 0, out $TRAF_HOME/monitor/test/dtmTest.lst} dtmCtrl $trace
 delay 3
 persist info dtm
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

 sleep 5
 monpstat
 Shutdown_status ${TestTitle} >> $PWD/dtmTest.lst
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     edb_pdsh -a grep PASSED $PWD/dtmTest.lst
     edb_pdsh -a grep FAILED $PWD/dtmTest.lst
 else
     grep PASSED $PWD/dtmTest.lst
     grep FAILED $PWD/dtmTest.lst
 fi
else
   if ( ([ $test '==' -1 ] || [ $test '==' 6 ]) ); then
     echo "DTM is not configured as a persistent process. Skipping test 6!"
   fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 7 ] ); then
Cleanup
Genconf 7
[ $? -eq 0 ] || return 101

TestTitle="Process Create Test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell <<eof
 startup
 delay 3
 exec {name \$PCRE8, nid 0, out $TRAF_HOME/monitor/test/procCreate.lst} procCreate $trace -x
 delay 3
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

 sleep 5
 monpstat
 Shutdown_status ${TestTitle} >> $PWD/procCreate.lst
 if [ "$cluster" = "-cluster" ]; then
     edb_pdsh -a grep PASSED $PWD/procCreate.lst
     edb_pdsh -a grep FAILED $PWD/procCreate.lst
 else
     grep PASSED $PWD/procCreate.lst
     grep FAILED $PWD/procCreate.lst
 fi
fi

if ( [ $test '==' -1 ] || [ $test '==' 8 ] ); then
Cleanup
Genconf 8
[ $? -eq 0 ] || return 101

TestTitle="Node Down Before Startup Test"
 echo "***"
 echo "*** Executing: ${TestTitle}"
 echo "***"
monshell<<eof 
 startup
 delay 3
 exec {nowait, name \$PCRE8, nid 0, out $TRAF_HOME/monitor/test/nodeDown.lst} procCreate $trace -y
 delay 3
 down 1 !
 delay 10
 !shutdown
 exit
eof

#monshell -nid 0 -c ps
#monshell -nid 0 -c ps monitor
#monshell -nid 0 -c ps;monshell -nid 1 -c ps;monshell -nid 2 -c ps;monshell -nid 3 -c ps;monshell -nid 4 -c ps;monshell -nid 5 -c ps
#monshell -nid 0 -c ps monitor;monshell -nid 1 -c ps monitor;monshell -nid 2 -c ps monitor;monshell -nid 3 -c ps monitor;monshell -nid 4 -c ps monitor;monshell -nid 5 -c ps monitor

monshell -a<<eof
 ps
 ps monitor
 shutdown
 exit
eof

 sleep 10
 monpstat
 Shutdown_status ${TestTitle} >> $PWD/nodeDown.lst
 cstat -h
 if [ "$cluster" = "-cluster" ]; then
     edb_pdsh -a grep PASSED $PWD/nodeDown.lst
     edb_pdsh -a grep FAILED $PWD/nodeDown.lst
 else
     grep PASSED $PWD/nodeDown.lst
     grep FAILED $PWD/nodeDown.lst
 fi
fi

if ( [ $test '==' -1 ] ); then
 echo "***"
 echo "*** Monitor Test Results"
 echo "***"
 if [ "$cluster" = "-cluster" ]; then
     edb_pdsh -a grep PASSED $PWD/childExit.lst
     edb_pdsh -a grep FAILED $PWD/childExit.lst
     edb_pdsh -a grep PASSED $PWD/multiNode.lst
     edb_pdsh -a grep FAILED $PWD/multiNode.lst
     edb_pdsh -a grep PASSED $PWD/regTest.lst
     edb_pdsh -a grep FAILED $PWD/regTest.lst
     edb_pdsh -a grep PASSED $PWD/deathNotice.lst
     edb_pdsh -a grep FAILED $PWD/deathNotice.lst
     edb_pdsh -a grep PASSED $PWD/persistentProc.lst
     edb_pdsh -a grep FAILED $PWD/persistentProc.lst
     edb_pdsh -a grep PASSED $PWD/dtmTest.lst
     edb_pdsh -a grep FAILED $PWD/dtmTest.lst
     edb_pdsh -a grep PASSED $PWD/procCreate.lst
     edb_pdsh -a grep FAILED $PWD/procCreate.lst
     edb_pdsh -a grep PASSED $PWD/nodeDown.lst
     edb_pdsh -a grep FAILED $PWD/nodeDown.lst
 else
     grep PASSED $PWD/childExit.lst
     grep FAILED $PWD/childExit.lst
     grep PASSED $PWD/multiNode.lst
     grep FAILED $PWD/multiNode.lst
     grep PASSED $PWD/regTest.lst
     grep FAILED $PWD/regTest.lst
     grep PASSED $PWD/deathNotice.lst
     grep FAILED $PWD/deathNotice.lst
     grep PASSED $PWD/persistentProc.lst
     grep FAILED $PWD/persistentProc.lst
     grep PASSED $PWD/dtmTest.lst
     grep FAILED $PWD/dtmTest.lst
     grep PASSED $PWD/procCreate.lst
     grep FAILED $PWD/procCreate.lst
     grep PASSED $PWD/nodeDown.lst
     grep FAILED $PWD/nodeDown.lst
 fi
fi
