--- mitgcm.org/scripts/check_outp 2009/06/17 21:01:44 1.12 +++ mitgcm.org/scripts/check_outp 2010/03/30 15:06:54 1.22 @@ -1,6 +1,6 @@ #! /usr/bin/env bash -# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.12 2009/06/17 21:01:44 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.22 2010/03/30 15:06:54 jmc Exp $ # # The purpose of this script is to compare # the latest output with the previous one (from the same platform with @@ -11,7 +11,7 @@ { echo echo "Usage: $0 [OPTIONS]" - echo + echo echo "where possible OPTIONS are:" echo " (-h|-help) print usage" echo " (-v|-verbose) verbose mode" @@ -28,10 +28,132 @@ echo " [def=\"$NBLDIFF\"]" echo " (-a |-addr )ADDRESS send summary to mail ADDRESS list" echo " [def=\"$ADDRESS\"]" - echo + echo exit 1 } +# create list of results dir for one platform: +make_resdir_list() +{ + if test $monthDir == 1 ; then + dir_list=`grep '.._'$mname ./dir_all` + else + dir_list=`( cd $INDIR ; ls -1 -t ../$PREV_P/??_*${mname}*/summary.txt 2> /dev/null | sed 's/\/summary.txt//' )` + #echo "dir_list='$dir_list'" + fi + for i in $dir_list ; do + + dir=$INDIR"/"$i + #--- get the opfile used to run testreport + OPTFILE= + if test -r $dir/summary.txt ; then + comm=`grep 'OPTFILE=' $dir/summary.txt` + eval $comm + fi + if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then + comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null` + eval $comm + fi + if test "x$OPTFILE" = x ; then + comm=`grep '^# OPTFILE=' $dir/*/Makefile* 2>/dev/null | head -1` + comm=${comm##*#} + eval $comm + fi + if test "x$OPTFILE" = x ; then + OPTFILE="not_explicitly_specified" + else + OPTFILE=${OPTFILE##*/} + fi + #--- case select test run by 1 user: get user who did run testreport + if test "x$sUser" != x ; then + tUser= + if test -f $dir/genmake_state ; then + nn=`sed -n '/^# executed by:/=' $dir/genmake_state` + if test "x$nn" != x ; then + nn=`expr $nn + 1` + tUser=`sed -n "$nn s/^# *// p" $dir/genmake_state | sed 's/@.*$//'` + fi + fi + if test "x$tUser" = "x" ; then + mkfile=`ls $dir/*/Makefile* 2>/dev/null | head -1` + if test "x$mkfile" = 'x' ; then + echo "no Makefile => discard test from: $i" | tee -a $OUTPFIL + continue + fi + nn=`sed -n '/^# executed by:/=' $mkfile` + if test "x$nn" != x ; then + nn=`expr $nn + 1` + tUser=`sed -n "$nn s/^# *// p" $mkfile | sed 's/@.*$//'` + fi + fi + if test "x$tUser" = x ; then + echo "no User found => discard test from: $i" | tee -a $OUTPFIL + continue + fi + if test $tUser != $sUser ; then + if test $dBug = 't' ; then + echo " discard test from: $i" | tee -a $OUTPFIL + fi + continue + fi + fi + + ADJOINT= + RESTART=0 + NOI3E=0 + MTH=0 + if test -r $dir/summary.txt ; then + comm=`grep 'ADJOINT=true' $dir/summary.txt 2>/dev/null` + eval $comm + RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt` + NOI3E=`grep -c "^run: .*testreport.* '*-noieee'*" $dir/summary.txt` + MTH=`grep -c "^run: .*testreport.* -mth" $dir/summary.txt` + fi + if test "x$RESTART" = x0 ; then + kind="forward" + test "x$ADJOINT" = xtrue && kind="adjoint" + else + kind="restart" + fi + if test "x$MTH" = x1 ; then + yy=`echo $OPTFILE | grep -c '+mth$'` + if test $yy = 0 ; then OPTFILE="${OPTFILE}+mth" ; fi + fi + if test "x$NOI3E" = x1 ; then + OPTFILE="${OPTFILE}.noieee" + fi + day=`echo $i | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` + ttt=`echo $day | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` + day=`echo $ttt | sed -e 's|_| |g' |awk '{print $NF}'` + + if [ $day -lt $FIRSTDAY ] ; then + echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./plist + elif [ $day -le $LASTDAY ] ; then + echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./slist + else + if test $dBug = 't' ; then + echo "discard day='$day' from: $i" | tee -a $OUTPFIL + fi + fi + + done + if test $dBug = 't' ; then + if test $monthDir == 1 ; then + echo "---- current list (n=$monthDir) :" | tee -a $OUTPFIL + cat ./slist | tee -a $OUTPFIL + fi + num1=`wc -l ./slist | awk '{print $1}'` + if test $num1 -gt 0 ; then + echo ".... previous list (n=$monthDir):" | tee -a $OUTPFIL + cat ./plist | tee -a $OUTPFIL | head -20 + fi + echo "----" | tee -a $OUTPFIL + fi +} + +#-------------------------------------------------------------------------- +# here starts the sequential part of the script: + #CURR_PER=`date +%Y`"_"`date +%m` CURR_DAY=`date +%Y%m%d` # defaults @@ -143,7 +265,7 @@ echo "INDIR='$INDIR'" >> $OUTPFIL echo "Checking latest output from $FIRSTDAY until $LASTDAY" >> $OUTPFIL -if test $dBug = 't' ; then +if test $dBug = 't' ; then echo "PERIOD='$PERIOD' , PREV_P='$PREV_P'" echo "FIRSTDAY='$FIRSTDAY' , LASTDAY='$LASTDAY'" fi @@ -178,15 +300,16 @@ ( cd $INDIR ; ls -1 -t */summary.txt | sed 's/\/summary.txt//' ) > ./dir_all if test "x$MACHINES" = "x_All_" ; then - MACHINES="faulks aces eddy bay meander lagoon hugo" - MACHINES="$MACHINES columbia edvir rays solasrv sx8 xd1" - MACHINES="$MACHINES beagle batsi starp" + MACHINES="faulks meander lagoon aces.jmc aces.ce107 dickens danton beagle harbor" + MACHINES="$MACHINES columbia pleiades iblade rays solssrv solasrv sx8" + MACHINES="$MACHINES trane dodongo dokdo bigred" MALL=`cat ./dir_all | sed -e 's|_| |g' | awk '{print $2}' | sort | uniq` for madd in $MALL ; do present=0 for m in $MACHINES ; do - echo $madd | grep $m > /dev/null 2>&1 + mm=`echo $m | sed 's/\./ /g' | awk '{print $1}'` + echo $madd | grep $mm > /dev/null 2>&1 RETVAL=$? test $RETVAL = 0 && present=1 continue @@ -194,106 +317,28 @@ test $present = 0 && MACHINES="$MACHINES $madd" done fi +#echo "MACHINES=$MACHINES" ; exit -for mname in $MACHINES ; do +for pname in $MACHINES ; do - if test $dBug = 't' ; then echo " $mname" ; fi - echo " name='$mname'" >> $OUTPFIL + mname=`echo $pname | sed 's/\./ /g' | awk '{print $1}'` + sUser= + if test $mname != $pname ; then + # to select results from user "sUser" + sUser=`echo $pname | sed 's/\./ /g' | awk '{print $2}'` + fi + + if test $dBug = 't' ; then echo " $pname" ; fi + echo " name='$pname'" >> $OUTPFIL ctot=0 ; cmis=0 ; cdif=0 - # put results from selected period in "slist", + # put results from selected period in "slist", # results from previous period in "plist" rm -f ./plist ./slist ; touch ./plist ./slist - for n in $listNum ; do - num1=`wc -l ./slist | awk '{print $1}'` - num0=`wc -l ./plist | awk '{print $1}'` - if test $n == 2 ; then - num2=`expr $num1 \* $n` - if [ $num0 -lt $num2 ] ; then - dir_list=`( cd $INDIR ; ls -1 -t ../$PREV_P/*${mname}*/summary.txt 2> /dev/null | sed 's/\/summary.txt//' )` - #echo "dir_list='$dir_list'" - fi - else - num2=$n - dir_list=`grep $mname ./dir_all` - fi - if [ $num0 -lt $num2 ] ; then - for i in $dir_list ; do - - dir=$INDIR"/"$i - OPTFILE= - if test -r $dir/summary.txt ; then - comm=`grep 'OPTFILE=' $dir/summary.txt` - eval $comm - OPTFILE=${OPTFILE##*/} - fi - if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then - comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null` - eval $comm - OPTFILE=${OPTFILE##*/} - fi - if test "x$OPTFILE" = x ; then - comm=`grep '^# OPTFILE=' $dir/*/Makefile* 2>/dev/null | head -1` - comm=${comm##*#} - eval $comm - OPTFILE=${OPTFILE##*/} - fi - if test "x$OPTFILE" = x ; then - OPTFILE="not_explicitly_specified" - fi - - ADJOINT= - RESTART=0 - NOI3E=0 - MTH=0 - if test -r $dir/summary.txt ; then - comm=`grep 'ADJOINT=true' $dir/summary.txt 2>/dev/null` - eval $comm - RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt` - NOI3E=`grep -c "^run: .*testreport.* '*-noieee'* " $dir/summary.txt` - MTH=`grep -c "^run: .*testreport.* -mth " $dir/summary.txt` - fi - if test "x$RESTART" = x0 ; then - kind="forward" - test "x$ADJOINT" = xtrue && kind="adjoint" - else - kind="restart" - fi - if test "x$NOI3E" = x1 ; then - OPTFILE="${OPTFILE}.noieee" - fi - if test "x$MTH" = x1 ; then - yy=`echo $OPTFILE | grep -c 'mth$'` - if test $yy = 0 ; then OPTFILE="${OPTFILE}+mth" ; fi - fi - day=`echo $i | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` - ttt=`echo $day | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` - day=`echo $ttt | sed -e 's|_| |g' |awk '{print $NF}'` - - if [ $day -lt $FIRSTDAY ] ; then - echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./plist - elif [ $day -le $LASTDAY ] ; then - echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./slist - else - if test $dBug = 't' ; then - echo "discard day='$day' from: $i" | tee -a $OUTPFIL - fi - fi - - done - if test $dBug = 't' ; then - if test $n == 1 ; then - echo "---- current list (n=$n) :" | tee -a $OUTPFIL - cat ./slist | tee -a $OUTPFIL - fi - echo ".... previous list (n=$n):" | tee -a $OUTPFIL - cat ./plist | tee -a $OUTPFIL | head -20 - fi - - fi - done - if test $dBug = 't' ; then echo "----" | tee -a $OUTPFIL ; fi + #- create list of results (slist & plist) from current month directory + monthDir=1 + make_resdir_list # Do we have any data? If so, create the latest pointer. num=`wc -l ./slist | awk '{print $1}'` @@ -305,18 +350,32 @@ ctot=`expr $ctot + 1` sline=`grep "^$key " ./slist | head -1` sdir=`echo $sline | cut -d " " -f 5` - kind=`echo $sline | cut -d " " -f 4` + type=`echo $sline | cut -d " " -f 4` optf=`echo $sline | cut -d " " -f 3` num=`grep -c "^$key" ./plist` + if test $num = 0 -a $monthDir = 1 ; then + #- add results from previous month directory to "plist". + monthDir=2 + make_resdir_list + num=`grep -c "^$key" ./plist` + fi #- discard unsafe test: - dd=`echo $optf | grep -c 'gfortran.*mth'` + dd=`echo $optf | grep -c 'gfortran+mth'` if test $mname = 'lagoon' -a $dd = 1 ; then num=-1 ; fi if test $mname = 'harbor' -a $dd = 1 ; then num=-1 ; fi - if test $mname = 'aces' -a $optf = 'linux_ia32_open64' ; then num=-1 ; fi - if test $mname = 'aces' -a $optf = 'linux_ia32_g95' ; then num=-1 ; fi - if test $num -gt 0 ; then + #if test $mname = 'aces' -a $optf = 'linux_ia32_open64' ; then num=-1 ; fi + #if test $mname = 'aces' -a $optf = 'linux_ia32_g95' ; then num=-1 ; fi + dd=`echo $optf | grep -c 'bigred.*+mth'` + if test $mname = 'bigred' -a $dd = 1 ; then num=-1 ; fi + if test $num -lt 0 ; then + echo " discard $sdir : $type , of='$optf'" | tee -a $OUTPFIL + cmis=`expr $cmis + 1` + elif test $num = '0' ; then + echo "no previous test for:" $key | tee -a $OUTPFIL + cmis=`expr $cmis + 1` + else pline=`grep "^$key " ./plist | head -1` - if test $dBug = 't' ; then + if test $dBug = 't' ; then echo "s='$sline'" | tee -a $OUTPFIL echo "p='$pline'" | tee -a $OUTPFIL fi @@ -330,18 +389,14 @@ | sed 's/ OPTFILE=.*\// OPTFILE=/g' \ | grep -v '^[A-S][a-t]* time: ' > tmpfp #-- re-order list of exp (according to local "sort" order) - listD=`grep '^[YN] [YN] [YN] [YN]' tmpfs \ - | sed 's/^[YN] .*pass //' | sed 's/^[YN] .*FAIL //' \ - | sed 's/^[YN] .*N\/O //' | sort` + listD=`grep '^[YN] [YN] [YN] [YN]' tmpfs | awk '{print $NF}' | sort` for xx in $listD do sed "/ $xx\$/d" tmpfs > tmpfn sed -n "/ $xx\$/p" tmpfs >> tmpfn mv tmpfn tmpfs done - listD=`grep '^[YN] [YN] [YN] [YN]' tmpfp \ - | sed 's/^[YN] .*pass //' | sed 's/^[YN] .*FAIL //' \ - | sed 's/^[YN] .*N\/O //' | sort` + listD=`grep '^[YN] [YN] [YN] [YN]' tmpfp | awk '{print $NF}' | sort` for xx in $listD do sed "/ $xx\$/d" tmpfp > tmpfn @@ -352,7 +407,7 @@ diff tmpfs tmpfp > tmpdf yy=$? if test $yy != '0' ; then - echo " $kind , of='$optf'" | tee -a $OUTPFIL + echo " $type , of='$optf'" | tee -a $OUTPFIL #grep '^run: ' $outp #grep '^run: ' $outs #-- score for each test: @@ -378,28 +433,23 @@ echo "< "`head -1 $outs`" ( $score_s )" >> $OUTPFIL echo "> "`head -1 $outp`" ( $score_p )" >> $OUTPFIL cat tmpdf >> $OUTPFIL - if [ $cdif -eq 0 ] ; then + if [ $cdif -eq 0 ] ; then clin=`printf '%3i (%5s,%5s)' $ndf $score_s $score_p` - else + else clin="$clin,"`printf '%3i (%5s,%5s)' $ndf $score_s $score_p` fi cdif=`expr $cdif + 1` echo '----------------------------------------' | tee -a $OUTPFIL else - echo "" | tee -a $OUTPFIL + echo " $type , of='$optf'" | tee -a $OUTPFIL fi rm -f tmpfs tmpfp tmpdf - elif test $num = '0' ; then - echo "no previous test for:" $key | tee -a $OUTPFIL - cmis=`expr $cmis + 1` - else - cmis=`expr $cmis + 1` fi done if [ $cmis -gt 0 ] ; then tmis='(-'$cmis')' ; else tmis='(--)' ; fi - printf '%11s : %3i %4s %3i ' $mname $ctot $tmis $cdif >> $OUTPSUM - if [ $cdif -gt 0 ] ; then - echo " $clin" >> $OUTPSUM + printf '%11s : %3i %4s %3i ' $pname $ctot $tmis $cdif >> $OUTPSUM + if [ $cdif -gt 0 ] ; then + echo " $clin" >> $OUTPSUM else echo "" >> $OUTPSUM ; fi fi