#!/bin/bash
#set -v
#Get the Test dir from the build Plan Name or from command line

set -o pipefail

TOP_DIR=`pwd`

# The first argument is the build plan name which contains
# plan/branch information. We are only interested in the
# last part of this name which is the test name.
TEST=`echo $1 | sed -r 's/.*Build and Test - ([0-9\.]+ - ){0,1}//' | awk '{ print $1; }'`
shift

TEST_DIR=test/core/$TEST
if [ ! -d "$TEST_DIR" ]; then
    echo "Test Directory does not exist. Probably the test $TEST is not checked in the branch"
    echo "Exiting with status 0 "
    exit 0
fi
cd $TEST_DIR

echo "TEST is $TEST"
pwd

PLANFILE=`mktemp plan.XXXXXX`

#Launch the test
./run-bamboo-test "$@" | tee $PLANFILE

STATUS=$?
if [ $STATUS != 0 ]; then
   echo "Workflow submission failed"
   exit $STATUS
fi

# check for bad planner outputs
if grep 'ERROR' $PLANFILE >/dev/null 2>&1; then
    echo "The planner logged a line with the string 'ERROR'"
    exit 1
fi

#Get RUNDIR from the planning output
RUN_DIR=`grep pegasus-remove $PLANFILE | awk '{print $5}'`
if [ "x$RUN_DIR" = "x" ]; then
    
    # if the workflow was planned with out --submit, it only has a pegasus-run line
    RUN_DIR=`grep pegasus-run $PLANFILE | awk '{print $2}'`

    if [ "x$RUN_DIR" = "x" ]; then
        echo "Unable to determine the RUN_DIR from the planner output - did the planner fail?" 1>&2
        exit 1
    fi
fi

echo "RUNDIR is $RUN_DIR"

# Change in to the rundir
cd $RUN_DIR

#Check status
$TOP_DIR/test/common/check-status
STATUS=$?

# always pegasus-analyzer to detect other problems
pegasus-analyzer $RUN_DIR
STATUS=$(($STATUS + $?))

if [ $STATUS = 0 ]; then
   # Run Pegasus Statistics
   pegasus-statistics -s all $RUN_DIR
   STATUS=$(($STATUS + $?))
fi

cd $RUN_DIR

# monitord checks
if [ ! -e monitord.done ]; then
    echo "monitord.done does not exist - did monitord finish successfully?"
    STATUS=$(($STATUS + 1))
fi

LOG_COUNT=`ls monitord.log* 2>/dev/null | wc -l`
if [ $LOG_COUNT -lt 1 -o $LOG_COUNT -gt 1 ]; then
    echo "$LOG_COUNT monitord log files found - did monitord restart?"
    STATUS=$(($STATUS + 1))
fi

if cat monitord.log | grep ERR >/dev/null 2>&1; then
    echo "monitord log contains errors:"
    echo
    cat monitord.log
    echo
    STATUS=$(($STATUS + 1))
fi

if cat monitord.log | grep WARN | grep -v -E '(read_stdout_stderr_files)|(unable to read error file)|(truncating std)' >/dev/null 2>&1; then
    echo "monitord log contains warnings:"
    echo
    cat monitord.log
    echo
    STATUS=$(($STATUS + 1))
fi

# check for metrics file consistency with submit files
METRICS_FILE=`ls *metrics 2>/dev/null|  grep -v dag`
METRICS_TOTAL_JOBS=`grep total_jobs $METRICS_FILE | sed -E "s/.*\"total_jobs\": //"`
NUM_SUBMIT_FILES=`ls *sub 2>/dev/null| grep -v condor.sub | wc -l | sed -E "s/\s//"`

if [ $METRICS_TOTAL_JOBS -ne $NUM_SUBMIT_FILES ]; then
    echo "Test failed, the number of submit files $NUM_SUBMIT_FILES does not match total_jobs value $METRICS_TOTAL_JOBS in metrics file $METRICS_FILE"
    STATUS=$(($STATUS + 1))
fi


if [ $STATUS -ne 0 ]; then
    echo "Test failed, creating tarball..."
    DIR="/nfs/ccg3/scratch/gideon/pegasus-failures"
    if ! [ -d "$DIR" ]; then
        DIR="/tmp"
    fi
    TARFILE=$(mktemp $DIR/pegasus-failure-XXXXXX.tar.gz)
    tar czf $TARFILE $RUN_DIR
    chmod 0644 $TARFILE
    echo "Created tarball $TARFILE"
fi

if [ $STATUS != 0 ]; then
   echo "ERROR: Test failed" 1>&2
   exit $STATUS
fi

exit

