--- mitgcm.org/scripts/check_outp 2008/08/27 21:19:32 1.1 +++ mitgcm.org/scripts/check_outp 2010/04/27 19:51:50 1.23 @@ -1,6 +1,6 @@ #! /usr/bin/env bash -# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.1 2008/08/27 21:19:32 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.23 2010/04/27 19:51:50 jmc Exp $ # # The purpose of this script is to compare # the latest output with the previous one (from the same platform with @@ -11,13 +11,14 @@ { echo echo "Usage: $0 [OPTIONS]" - echo + echo echo "where possible OPTIONS are:" echo " (-h|-help) print usage" echo " (-v|-verbose) verbose mode" echo " (-l |-list )MACHINES check platforms from this list" echo " [def=\"$MACHINES\"]" echo " (-d |-day )FIRSTDAY select output from day=\"YYYYMMDD\"" + echo " (argument of \"date -d\" is allowed)" echo " [def=\"$FIRSTDAY\"]" echo " (-u |-upto )LASTDAY until last day=\"YYYYMMDD\" (=selected period)" echo " [def=\"FIRSTDAY\"]" @@ -27,10 +28,132 @@ echo " [def=\"$NBLDIFF\"]" echo " (-a |-addr )ADDRESS send summary to mail ADDRESS list" echo " [def=\"$ADDRESS\"]" - echo + echo exit 1 } +# create list of results dir for one platform: +make_resdir_list() +{ + if test $monthDir == 1 ; then + dir_list=`grep '.._'$mname ./dir_all` + else + dir_list=`( cd $INDIR ; ls -1 -t ../$PREV_P/??_*${mname}*/summary.txt 2> /dev/null | sed 's/\/summary.txt//' )` + #echo "dir_list='$dir_list'" + fi + for i in $dir_list ; do + + dir=$INDIR"/"$i + #--- get the opfile used to run testreport + OPTFILE= + if test -r $dir/summary.txt ; then + comm=`grep 'OPTFILE=' $dir/summary.txt` + eval $comm + fi + if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then + comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null` + eval $comm + fi + if test "x$OPTFILE" = x ; then + comm=`grep '^# OPTFILE=' $dir/*/Makefile* 2>/dev/null | head -1` + comm=${comm##*#} + eval $comm + fi + if test "x$OPTFILE" = x ; then + OPTFILE="not_explicitly_specified" + else + OPTFILE=${OPTFILE##*/} + fi + #--- case select test run by 1 user: get user who did run testreport + if test "x$sUser" != x ; then + tUser= + if test -f $dir/genmake_state ; then + nn=`sed -n '/^# executed by:/=' $dir/genmake_state` + if test "x$nn" != x ; then + nn=`expr $nn + 1` + tUser=`sed -n "$nn s/^# *// p" $dir/genmake_state | sed 's/@.*$//'` + fi + fi + if test "x$tUser" = "x" ; then + mkfile=`ls $dir/*/Makefile* 2>/dev/null | head -1` + if test "x$mkfile" = 'x' ; then + echo "no Makefile => discard test from: $i" | tee -a $OUTPFIL + continue + fi + nn=`sed -n '/^# executed by:/=' $mkfile` + if test "x$nn" != x ; then + nn=`expr $nn + 1` + tUser=`sed -n "$nn s/^# *// p" $mkfile | sed 's/@.*$//'` + fi + fi + if test "x$tUser" = x ; then + echo "no User found => discard test from: $i" | tee -a $OUTPFIL + continue + fi + if test $tUser != $sUser ; then + if test $dBug = 't' ; then + echo " discard test from: $i" | tee -a $OUTPFIL + fi + continue + fi + fi + + ADJOINT= + RESTART=0 + NOI3E=0 + MTH=0 + if test -r $dir/summary.txt ; then + comm=`grep 'ADJOINT=true' $dir/summary.txt 2>/dev/null` + eval $comm + RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt` + NOI3E=`grep -c "^run: .*testreport.* '*-noieee'*" $dir/summary.txt` + MTH=`grep -c "^run: .*testreport.* -mth" $dir/summary.txt` + fi + if test "x$RESTART" = x0 ; then + kind="forward" + test "x$ADJOINT" = xtrue && kind="adjoint" + else + kind="restart" + fi + if test "x$MTH" = x1 ; then + yy=`echo $OPTFILE | grep -c '+mth$'` + if test $yy = 0 ; then OPTFILE="${OPTFILE}+mth" ; fi + fi + if test "x$NOI3E" = x1 ; then + OPTFILE="${OPTFILE}.noieee" + fi + day=`echo $i | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` + ttt=`echo $day | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` + day=`echo $ttt | sed -e 's|_| |g' |awk '{print $NF}'` + + if [ $day -lt $FIRSTDAY ] ; then + echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./plist + elif [ $day -le $LASTDAY ] ; then + echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./slist + else + if test $dBug = 't' ; then + echo "discard day='$day' from: $i" | tee -a $OUTPFIL + fi + fi + + done + if test $dBug = 't' ; then + if test $monthDir == 1 ; then + echo "---- current list (n=$monthDir) :" | tee -a $OUTPFIL + cat ./slist | tee -a $OUTPFIL + fi + num1=`wc -l ./slist | awk '{print $1}'` + if test $num1 -gt 0 ; then + echo ".... previous list (n=$monthDir):" | tee -a $OUTPFIL + cat ./plist | tee -a $OUTPFIL | head -20 + fi + echo "----" | tee -a $OUTPFIL + fi +} + +#-------------------------------------------------------------------------- +# here starts the sequential part of the script: + #CURR_PER=`date +%Y`"_"`date +%m` CURR_DAY=`date +%Y%m%d` # defaults @@ -42,6 +165,8 @@ ADDRESS='none' dBug=f sTime=`date` +#- to get case insensitive "ls" (and order of tested experiments) +export LC_ALL="en_US.UTF-8" # Parse options ac_prev= @@ -101,6 +226,16 @@ esac done +#-- test FIRSTDAY content ; interpret as "date -d" arg. if not YYYYMMDD +ttt=`echo "y$FIRSTDAY" | sed 's/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]/z/'` +if test "x$ttt" != 'xyz' ; then + FIRSTDAY=`date -d "$FIRSTDAY" +%Y%m%d` +fi +ttt=`echo "y$FIRSTDAY" | sed 's/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]/z/'` +if test "x$ttt" != 'xyz' ; then + echo "Error: FIRSTDAY='$FIRSDAY' not recognized" + exit 2 +fi if test $LASTDAY = "30000000" ; then LASTDAY=$FIRSTDAY ; fi CMDLINE=$0 for xx in "$@" ; do nw=`echo $xx | wc -w` @@ -130,7 +265,7 @@ echo "INDIR='$INDIR'" >> $OUTPFIL echo "Checking latest output from $FIRSTDAY until $LASTDAY" >> $OUTPFIL -if test $dBug = 't' ; then +if test $dBug = 't' ; then echo "PERIOD='$PERIOD' , PREV_P='$PREV_P'" echo "FIRSTDAY='$FIRSTDAY' , LASTDAY='$LASTDAY'" fi @@ -144,7 +279,7 @@ fi else echo "$INDIR is not a directory" - exit + exit 3 fi #- a short summary of this checking : @@ -165,15 +300,16 @@ ( cd $INDIR ; ls -1 -t */summary.txt | sed 's/\/summary.txt//' ) > ./dir_all if test "x$MACHINES" = "x_All_" ; then - MACHINES="faulks aces eddy bay meander lagoon hugo" - MACHINES="$MACHINES columbia edvir rays sx8 xd1" - MACHINES="$MACHINES beagle batsi starp" + MACHINES="faulks meander lagoon aces.jmc aces.ce107 dickens danton beagle harbor" + MACHINES="$MACHINES columbia pleiades iblade rays solssrv solasrv sx8" + MACHINES="$MACHINES trane dodongo dokdo bigred" MALL=`cat ./dir_all | sed -e 's|_| |g' | awk '{print $2}' | sort | uniq` for madd in $MALL ; do present=0 for m in $MACHINES ; do - echo $madd | grep $m > /dev/null 2>&1 + mm=`echo $m | sed 's/\./ /g' | awk '{print $1}'` + echo $madd | grep $mm > /dev/null 2>&1 RETVAL=$? test $RETVAL = 0 && present=1 continue @@ -181,100 +317,28 @@ test $present = 0 && MACHINES="$MACHINES $madd" done fi +#echo "MACHINES=$MACHINES" ; exit -for mname in $MACHINES ; do +for pname in $MACHINES ; do + + mname=`echo $pname | sed 's/\./ /g' | awk '{print $1}'` + sUser= + if test $mname != $pname ; then + # to select results from user "sUser" + sUser=`echo $pname | sed 's/\./ /g' | awk '{print $2}'` + fi - if test $dBug = 't' ; then echo " $mname" ; fi - echo " name='$mname'" >> $OUTPFIL + if test $dBug = 't' ; then echo " $pname" ; fi + echo " name='$pname'" >> $OUTPFIL ctot=0 ; cmis=0 ; cdif=0 - # put results from selected period in "slist", + # put results from selected period in "slist", # results from previous period in "plist" rm -f ./plist ./slist ; touch ./plist ./slist - for n in $listNum ; do - num1=`wc -l ./slist | awk '{print $1}'` - num0=`wc -l ./plist | awk '{print $1}'` - if test $n == 2 ; then - num2=`expr $num1 \* $n` - if [ $num0 -lt $num2 ] ; then - dir_list=`( cd $INDIR ; ls -1 -t ../$PREV_P/*${mname}*/summary.txt | sed 's/\/summary.txt//' )` - #echo "dir_list='$dir_list'" - fi - else - num2=$n - dir_list=`grep $mname ./dir_all` - fi - if [ $num0 -lt $num2 ] ; then - for i in $dir_list ; do - - dir=$INDIR"/"$i - OPTFILE= - if test -r $dir/summary.txt ; then - comm=`grep 'OPTFILE=' $dir/summary.txt` - eval $comm - OPTFILE=${OPTFILE##*/} - fi - if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then - comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null` - eval $comm - OPTFILE=${OPTFILE##*/} - fi - if test "x$OPTFILE" = x ; then - comm=`grep '^# OPTFILE=' $dir/*/Makefile* 2>/dev/null | head -1` - comm=${comm##*#} - eval $comm - OPTFILE=${OPTFILE##*/} - fi - if test "x$OPTFILE" = x ; then - OPTFILE="not_explicitly_specified" - fi - - ADJOINT= - RESTART=0 - NOI3E=0 - if test -r $dir/summary.txt ; then - comm=`grep 'ADJOINT=true' $dir/summary.txt 2>/dev/null` - eval $comm - RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt` - NOI3E=`grep -c "^run: .*testreport .* '-noieee' " $dir/summary.txt` - fi - if test "x$RESTART" = x0 ; then - kind="forward" - test "x$ADJOINT" = xtrue && kind="adjoint" - else - kind="restart" - fi - if test "x$NOI3E" = x1 ; then - OPTFILE="${OPTFILE}.noieee" - fi - day=`echo $i | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` - ttt=`echo $day | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` - day=`echo $ttt | sed -e 's|_| |g' |awk '{print $NF}'` - - if [ $day -lt $FIRSTDAY ] ; then - echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./plist - elif [ $day -le $LASTDAY ] ; then - echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./slist - else - if test $dBug = 't' ; then - echo "discard day='$day' from: $i" | tee -a $OUTPFIL - fi - fi - - done - if test $dBug = 't' ; then - if test $n == 1 ; then - echo "---- current list (n=$n) :" | tee -a $OUTPFIL - cat ./slist | tee -a $OUTPFIL - fi - echo ".... previous list (n=$n):" | tee -a $OUTPFIL - cat ./plist | tee -a $OUTPFIL - fi - - fi - done - if test $dBug = 't' ; then echo "----" | tee -a $OUTPFIL ; fi + #- create list of results (slist & plist) from current month directory + monthDir=1 + make_resdir_list # Do we have any data? If so, create the latest pointer. num=`wc -l ./slist | awk '{print $1}'` @@ -286,30 +350,64 @@ ctot=`expr $ctot + 1` sline=`grep "^$key " ./slist | head -1` sdir=`echo $sline | cut -d " " -f 5` - kind=`echo $sline | cut -d " " -f 4` + type=`echo $sline | cut -d " " -f 4` optf=`echo $sline | cut -d " " -f 3` num=`grep -c "^$key" ./plist` + if test $num = 0 -a $monthDir = 1 ; then + #- add results from previous month directory to "plist". + monthDir=2 + make_resdir_list + num=`grep -c "^$key" ./plist` + fi #- discard unsafe test: - dd=`echo $optf | grep -c 'gfortran.*mth'` - if [ $dd -ge 1 ] ; then num=-1 ; fi - if test $num -gt 0 ; then + dd=`echo $optf | grep -c 'gfortran+mth'` + if test $mname = 'lagoon' -a $dd = 1 ; then num=-1 ; fi + if test $mname = 'harbor' -a $dd = 1 ; then num=-1 ; fi + #if test $mname = 'aces' -a $optf = 'linux_ia32_open64' ; then num=-1 ; fi + #if test $mname = 'aces' -a $optf = 'linux_ia32_g95' ; then num=-1 ; fi + dd=`echo $optf | grep -c 'bigred.*noieee'` + if test $mname = 'bigred' -a $dd = 1 ; then num=-1 ; fi + if test $num -lt 0 ; then + echo " discard $sdir : $type , of='$optf'" | tee -a $OUTPFIL + cmis=`expr $cmis + 1` + elif test $num = '0' ; then + echo "no previous test for:" $key | tee -a $OUTPFIL + cmis=`expr $cmis + 1` + else pline=`grep "^$key " ./plist | head -1` - if test $dBug = 't' ; then + if test $dBug = 't' ; then echo "s='$sline'" | tee -a $OUTPFIL echo "p='$pline'" | tee -a $OUTPFIL fi pdir=`echo $pline | cut -d " " -f 5` outs="$INDIR/$sdir/summary.txt" sed -n "5,$ p" $outs | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' \ + | sed 's/ OPTFILE=.*\// OPTFILE=/g' \ | grep -v '^[A-S][a-t]* time: ' > tmpfs outp="$INDIR/$pdir/summary.txt" sed -n "5,$ p" $outp | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' \ + | sed 's/ OPTFILE=.*\// OPTFILE=/g' \ | grep -v '^[A-S][a-t]* time: ' > tmpfp + #-- re-order list of exp (according to local "sort" order) + listD=`grep '^[YN] [YN] [YN] [YN]' tmpfs | awk '{print $NF}' | sort` + for xx in $listD + do + sed "/ $xx\$/d" tmpfs > tmpfn + sed -n "/ $xx\$/p" tmpfs >> tmpfn + mv tmpfn tmpfs + done + listD=`grep '^[YN] [YN] [YN] [YN]' tmpfp | awk '{print $NF}' | sort` + for xx in $listD + do + sed "/ $xx\$/d" tmpfp > tmpfn + sed -n "/ $xx\$/p" tmpfp >> tmpfn + mv tmpfn tmpfp + done echo -n "=== diff $sdir $pdir :" | tee -a $OUTPFIL diff tmpfs tmpfp > tmpdf yy=$? if test $yy != '0' ; then - echo " $kind , of='$optf'" | tee -a $OUTPFIL + echo " $type , of='$optf'" | tee -a $OUTPFIL #grep '^run: ' $outp #grep '^run: ' $outs #-- score for each test: @@ -335,28 +433,23 @@ echo "< "`head -1 $outs`" ( $score_s )" >> $OUTPFIL echo "> "`head -1 $outp`" ( $score_p )" >> $OUTPFIL cat tmpdf >> $OUTPFIL - if [ $cdif -eq 0 ] ; then + if [ $cdif -eq 0 ] ; then clin=`printf '%3i (%5s,%5s)' $ndf $score_s $score_p` - else + else clin="$clin,"`printf '%3i (%5s,%5s)' $ndf $score_s $score_p` fi cdif=`expr $cdif + 1` echo '----------------------------------------' | tee -a $OUTPFIL else - echo "" | tee -a $OUTPFIL + echo " $type , of='$optf'" | tee -a $OUTPFIL fi rm -f tmpfs tmpfp tmpdf - elif test $num = '0' ; then - echo "no previous test for:" $key | tee -a $OUTPFIL - cmis=`expr $cmis + 1` - else - cmis=`expr $cmis + 1` fi done if [ $cmis -gt 0 ] ; then tmis='(-'$cmis')' ; else tmis='(--)' ; fi - printf '%11s : %3i %4s %3i ' $mname $ctot $tmis $cdif >> $OUTPSUM - if [ $cdif -gt 0 ] ; then - echo " $clin" >> $OUTPSUM + printf '%11s : %3i %4s %3i ' $pname $ctot $tmis $cdif >> $OUTPSUM + if [ $cdif -gt 0 ] ; then + echo " $clin" >> $OUTPSUM else echo "" >> $OUTPSUM ; fi fi @@ -373,4 +466,4 @@ fi echo ".. cat $OUTPSUM" cat $OUTPSUM - +exit 0