--- mitgcm.org/scripts/check_outp 2013/04/05 21:15:52 1.38 +++ mitgcm.org/scripts/check_outp 2022/12/03 16:59:30 1.56 @@ -1,6 +1,6 @@ #! /usr/bin/env bash -# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.38 2013/04/05 21:15:52 jmc Exp $ +# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.56 2022/12/03 16:59:30 jmc Exp $ # # The purpose of this script is to compare # the latest output with the previous one (from the same platform with @@ -39,7 +39,7 @@ make_resdir_list() { if test $monthDir == 1 ; then - dir_list=`grep '.._'$mname ./dir_all` + dir_list=`grep '.._'$mname $TMP.dir_all` else dir_list=`( cd $INDIR ; ls -1 -t ../$PREV_P/??_*${mname}*/summary.txt 2> /dev/null | sed 's/\/summary.txt//' )` #echo "dir_list='$dir_list'" @@ -105,6 +105,7 @@ TANGLIN=0 OPENAD=0 RESTART=0 + Special=0 EXTRA= FAST=0 DVLP=0 @@ -121,21 +122,31 @@ OPENAD=`grep -c '^TangLin .* OpenAD' $dir/summary.txt` fi RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt` + #- special label for "darwin" test + Special=`echo $i | grep -c '\-darwin[0-9]'` + if test $Special != 0 ; then + Special=`echo $i | sed -e "s/.*$mname/$mname/" -e "s/_/ /g" -e "s/-/ /g"` + if test $dBug = 't' ; then echo -n " Special='$Special' ;" | tee -a $OUTPFIL ; fi + Special=`echo $Special | awk '{print $2}' | sed 's/darwin//'` + fi comm=`grep '^run: .*testreport.* ' $dir/summary.txt` EXTRA=`echo "$comm" | grep " -*-tdir\>" | sed -e "s/^.* -*-tdir\>//" -e "s/ -.*$//"` if test "x$EXTRA" = x ; then EXTRA=`echo "$comm" | grep " -*-t\>" | sed -e "s/^.*-*-t\>//" -e "s/ -.*$//"` fi if test "x$EXTRA" = x ; then EXTRA=0 ; else - #echo -n "EXTRA=$EXTRA" + #if test $dBug = 't' ; then echo -n " EXTRA='$EXTRA' ;" | tee -a $OUTPFIL ; fi nn0=`echo $EXTRA | sed "s/ *' *//g" | wc -w` - nn1=`echo $EXTRA | sed "s/ *' *//g" | tr ' ' '\n' | grep -c "\"` if test "x$FAST" = x0 ; then FAST=`echo "$comm" | grep -c " '*-noieee'*"` @@ -150,7 +161,7 @@ if test "x$UR4" = x0 ; then UR4=`echo "$comm" | grep -c " -*-ur4\>"` fi - fi + else EXTRA=0 ; fi if test "x$ADJOINT" = x1 ; then kind="adm-TAF" ; order=0 if test "x$OPENAD" = x1 ; then kind="adm-OAD" ; order=2 ; fi @@ -162,14 +173,14 @@ else kind="restart" ; order=5 fi - order=`expr $order + 10 \* $EXTRA` - order=`printf '%3.3i' $order` if test "x$UR4" = x1 ; then OPTFILE="${OPTFILE}.use_r4" fi if test "x$MPI" = x1 ; then yy=`echo $OPTFILE | grep -c '+mpi'` - if test $yy = 0 ; then OPTFILE="${OPTFILE}+mpi" ; fi + if test $yy = 0 ; then yy=`echo $OPTFILE | grep -c '+impi'` ; fi + if test $yy = 0 ; then OPTFILE="${OPTFILE}+mpi" + else order=`expr $order + 200` ; fi fi if test "x$MTH" = x1 ; then yy=`echo $OPTFILE | grep -c '+mth$'` @@ -181,14 +192,20 @@ if test "x$DVLP" = x1 ; then OPTFILE="${OPTFILE}.dvlp" fi + if [ $Special -eq 0 ] ; then + order=`expr $order + 10 \* $EXTRA` + else + order=`expr $order + 10 \* $Special` + fi + order=`printf '%3.3i' $order` day=`echo $i | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` ttt=`echo $day | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'` day=`echo $ttt | sed -e 's|_| |g' |awk '{print $NF}'` if [ $day -lt $BEFORE ] ; then - echo "$OPTFILE$order $day $OPTFILE $kind $i" >> ./plist + echo "$OPTFILE$order $day $OPTFILE $kind $i" >> $TMP.plist elif [ $day -ge $FIRSTDAY -a $day -le $LASTDAY ] ; then - echo "$OPTFILE$order $day $OPTFILE $kind $i" >> ./slist + echo "$OPTFILE$order $day $OPTFILE $kind $i" >> $TMP.slist else if test $dBug = 't' ; then echo "discard day='$day' from: $i" | tee -a $OUTPFIL @@ -199,14 +216,14 @@ if test $dBug = 't' ; then if test $monthDir == 1 ; then echo "---- current list (n=$monthDir) :" | tee -a $OUTPFIL - cat ./slist | tee -a $OUTPFIL + cat $TMP.slist | tee -a $OUTPFIL fi - num1=`wc -l ./slist | awk '{print $1}'` + num1=`wc -l $TMP.slist | awk '{print $1}'` if test $num1 -gt 0 ; then echo ".... previous list (n=$monthDir):" | tee -a $OUTPFIL - cat ./plist | tee -a $OUTPFIL | head -20 + cat $TMP.plist | tee -a $OUTPFIL | head -20 fi - echo "----" | tee -a $OUTPFIL + echo "----" | tee -a $OUTPFIL fi } @@ -224,6 +241,7 @@ OUTPFIL=`basename $0`'.log' NBLDIFF=5 ADDRESS='none' +MAILCMD="mail" dBug=f t4update=0 sTime=`date` @@ -242,7 +260,6 @@ fi ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'` - case $ac_option in -help | --help | -h | --h) usage ;; @@ -272,10 +289,22 @@ *) echo "Error: don't understand argument \"$ac_option\"" usage ;; - esac done + +#TMP=./chkoutp_$$ +#- try to put temporary files in system-local /tmp dir +TMP=/tmp/chkoutp_$$ +touch $TMP ; retVal=$? +if [ $retVal -eq 0 ] ; then + if test ! -r $TMP ; then TMP=./chkoutp_$$ ; fi +else + TMP=./chkoutp_$$ +fi +rm -f $TMP +if test $dBug = 't' ; then echo "temp files: $TMP" ; fi + #-- test FIRSTDAY content ; interpret as "date -d" arg. if not YYYYMMDD ttt=`echo "y$FIRSTDAY" | sed 's/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]/z/'` if test "x$ttt" != 'xyz' ; then @@ -302,9 +331,9 @@ PREV_P="${PY}_${PM}"; PERIOD=`echo $PERIOD | sed 's/^[0-9][0-9][0-9][0-9]/&_/'` -#INDIR="/net/orwell/export/export-9/mitgcm-testing/results/$PERIOD" +#INDIR="/net/zany.mit.edu/data/ORWELL/export-9/mitgcm-testing/results/$PERIOD" INDIR="/u/u0/httpd/html/testing/results/$PERIOD" -#INDIR="/export/export-7/u/u2/jmc/mitgcm/test_web/results/$PERIOD" +#INDIR="$HOME/mitgcm/test_web/results/$PERIOD" #- a short summary of this checking : OUTPSUM=`echo $LASTDAY | sed 's/^20../_/'` @@ -319,15 +348,20 @@ if test $PRC_MAILS -ot $OUTPSUM ; then echo ' '$CMDLINE | tee -a $OUTPSUM echo " t4update: skip "`basename $0`" ( $PRC_MAILS older than $OUTPSUM )" | tee -a $OUTPSUM - echo "End time: "`date` | tee -a $OUTPSUM + echo " End time: "`date` | tee -a $OUTPSUM exit 0 else if test -e $OUTPFIL ; then mv -f $OUTPFIL $OUTPFIL'_bak' ; fi echo " t4update: run "`basename $0`" ( $PRC_MAILS newer than $OUTPSUM )" | tee $OUTPFIL fi + elif test -f $OUTPSUM -a $FIRSTDAY = $CURR_DAY ; then + echo ' '$CMDLINE | tee -a $OUTPSUM + echo " t4update: skip "`basename $0`" ( $PRC_MAILS missing )" | tee -a $OUTPSUM + echo " End time: "`date` | tee -a $OUTPSUM + exit 0 else if test -e $OUTPFIL ; then mv -f $OUTPFIL $OUTPFIL'_bak' ; fi - echo " t4update ignored (not current day or $PRC_MAILS or $OUTPSUM missing)" | tee $OUTPFIL + echo " t4update ignored (not current day or $OUTPSUM missing)" | tee $OUTPFIL fi elif test -e $OUTPFIL ; then mv -f $OUTPFIL $OUTPFIL'_bak' @@ -372,13 +406,13 @@ echo " Machine Tot. Nb. no Nb with Nb diff" >> $OUTPSUM echo " name checked comp Diff lines" >> $OUTPSUM -( cd $INDIR ; ls -1 -t */summary.txt | sed 's/\/summary.txt//' ) > ./dir_all +( cd $INDIR ; ls -1 -t */summary.txt | sed 's/\/summary.txt//' ) > $TMP.dir_all if test "x$MACHINES" = "x_All_" ; then - MACHINES="aces- acesgrid baudelaire dickens danton beagle" - MACHINES="$MACHINES pleiades iblade sx8 uv100 solasrv stomp weber" + MACHINES="villon batsi engaging1 engaging- svante glacier" + MACHINES="$MACHINES ollie albedo pleiades archer" - MALL=`cat ./dir_all | sed -e 's|_| |g' | awk '{print $2}' | sort | uniq` + MALL=`cat $TMP.dir_all | sed -e 's|_| |g' | awk '{print $2}' | sort | uniq` for madd in $MALL ; do present=0 for m in $MACHINES ; do @@ -407,37 +441,37 @@ echo " name='$pname'" >> $OUTPFIL ctot=0 ; cmis=0 ; cdif=0 - # put results from selected period in "slist", - # results from previous period in "plist" - rm -f ./plist ./slist ; touch ./plist ./slist + # put results from selected period in "TMP.slist", + # results from previous period in "TMP.plist" + rm -f $TMP.plist $TMP.slist ; touch $TMP.plist $TMP.slist #- create list of results (slist & plist) from current month directory monthDir=1 make_resdir_list # Do we have any data? If so, create the latest pointer. - num=`wc -l ./slist | awk '{print $1}'` + num=`wc -l $TMP.slist | awk '{print $1}'` if test $num -gt 0 ; then - keys=`cat ./slist | cut -d " " -f 1 | sort | uniq` + keys=`cat $TMP.slist | cut -d " " -f 1 | sort | uniq` for key in $keys ; do - ctot=`expr $ctot + 1` - sline=`grep "^$key " ./slist | head -1` + ctot=`expr $ctot + 1` + sline=`grep "^$key " $TMP.slist | head -1` sdir=`echo $sline | cut -d " " -f 5` type=`echo $sline | cut -d " " -f 4` optf=`echo $sline | cut -d " " -f 3` - num=`grep -c "^$key" ./plist` + num=`grep -c "^$key" $TMP.plist` if test $num = 0 -a $monthDir = 1 ; then #- add results from previous month directory to "plist". monthDir=2 make_resdir_list - num=`grep -c "^$key" ./plist` + num=`grep -c "^$key" $TMP.plist` fi #- discard unsafe test: dd=`echo $optf | grep -c 'gfortran+mth'` - if test $mname = 'lagoon' -a $dd = 1 ; then num=-1 ; fi - if test $mname = 'harbor' -a $dd = 1 ; then num=-1 ; fi + #if test $mname = 'lagoon' -a $dd = 1 ; then num=-1 ; fi + #if test $mname = 'harbor' -a $dd = 1 ; then num=-1 ; fi #if test $mname = 'aces' -a $optf = 'linux_ia32_open64' ; then num=-1 ; fi #if test $mname = 'aces' -a $optf = 'linux_ia32_g95' ; then num=-1 ; fi dd=`echo $optf | grep -c 'bigred.*noieee'` @@ -446,10 +480,11 @@ echo " discard $sdir : $type , of='$optf'" | tee -a $OUTPFIL cmis=`expr $cmis + 1` elif test $num = '0' ; then - echo "no previous test for:" $key | tee -a $OUTPFIL + echo " dir='$sdir', type='$type', of='$optf':" | tee -a $OUTPFIL + echo " --> no previous test matching key='$key'" | tee -a $OUTPFIL cmis=`expr $cmis + 1` else - pline=`grep "^$key " ./plist | head -1` + pline=`grep "^$key " $TMP.plist | head -1` if test $dBug = 't' ; then echo "s='$sline'" | tee -a $OUTPFIL echo "p='$pline'" | tee -a $OUTPFIL @@ -458,56 +493,69 @@ outs="$INDIR/$sdir/summary.txt" xx=`( echo 5 ; sed -n '/^ OPTFILE=/=' $outs ) | tail -1` sed "1,$xx d" $outs | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' \ - | grep -v '^[A-S][a-t]* time: ' > tmpfs + | grep -v '^[A-S][a-t]* time: ' > $TMP.fs outp="$INDIR/$pdir/summary.txt" xx=`( echo 5 ; sed -n '/^ OPTFILE=/=' $outp ) | tail -1` sed "1,$xx d" $outp | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' \ - | grep -v '^[A-S][a-t]* time: ' > tmpfp - #-- re-order list of exp (according to local "sort" order) - listD=`grep '^[YN] [YN] [YN] [YN]' tmpfs | awk '{print $NF}' | sort` - for xx in $listD - do - sed "/ $xx\$/d" tmpfs > tmpfn - sed -n "/ $xx\$/p" tmpfs >> tmpfn - mv tmpfn tmpfs - done - listD=`grep '^[YN] [YN] [YN] [YN]' tmpfp | awk '{print $NF}' | sort` - for xx in $listD - do - sed "/ $xx\$/d" tmpfp > tmpfn - sed -n "/ $xx\$/p" tmpfp >> tmpfn - mv tmpfn tmpfp - done + | grep -v '^[A-S][a-t]* time: ' > $TMP.fp + #-- check if list of exp are the same (and in the same order) + sed 's/ (e=.*, w=.*)$//' $TMP.fs > $TMP.fn + listDs=`grep '^[YN] [YN] [YN] [YN]' $TMP.fn | awk '{print $NF}'` + #mv -f $TMP.fn $TMP.fs + rm -f $TMP.fn + sed 's/ (e=.*, w=.*)$//' $TMP.fp > $TMP.fn + listDp=`grep '^[YN] [YN] [YN] [YN]' $TMP.fn | awk '{print $NF}'` + #mv -f $TMP.fn $TMP.fp + rm -f $TMP.fn + #echo "listDs='$listDs'" ; echo "listDp='$listDp'" + if test "$listDs" != "$listDp" ; then + if test $dBug = 't' ; then echo ' -> re-order list of exp' ; fi + #-- re-order list of exp (according to local "sort" order) + listDs=`echo "$listDs" | sort` + for xx in $listDs ; do + sed -e "/ $xx\$/d" -e "/ $xx /d" $TMP.fs > $TMP.fn + sed -n -e "/ $xx\$/p" -e "/ $xx /p" $TMP.fs >> $TMP.fn + mv -f $TMP.fn $TMP.fs + done + listDp=`echo "$listDp" | sort` + for xx in $listDp ; do + sed -e "/ $xx\$/d" -e "/ $xx /d" $TMP.fp > $TMP.fn + sed -n -e "/ $xx\$/p" -e "/ $xx /p" $TMP.fp >> $TMP.fn + mv -f $TMP.fn $TMP.fp + done + #else + # echo 'same list of exp => skip re-ordering' + fi echo -n "=== diff $sdir $pdir :" | tee -a $OUTPFIL - diff tmpfs tmpfp > tmpdf + diff $TMP.fs $TMP.fp > $TMP.dif yy=$? if test $yy != '0' ; then echo " $type , of='$optf'" | tee -a $OUTPFIL #grep '^run: ' $outp #grep '^run: ' $outs #-- score for each test: - grep '^[YN] [YN] [YN] [YN]' tmpfs > tmploc 2>/dev/null + grep '^[YN] [YN] [YN] [YN]' $TMP.fs > tmploc 2>/dev/null t_tot=`cat tmploc | wc -l | sed -e 's| ||g'` t_pass=`grep '^Y Y Y Y' tmploc | grep 'pass ' | wc -l | sed -e 's| ||g'` score_s=`printf '%2i' $t_pass`":$t_tot" - grep '^[YN] [YN] [YN] [YN]' tmpfp > tmploc 2>/dev/null + grep '^[YN] [YN] [YN] [YN]' $TMP.fp > tmploc 2>/dev/null t_tot=`cat tmploc | wc -l | sed -e 's| ||g'` t_pass=`grep '^Y Y Y Y' tmploc | grep 'pass ' | wc -l | sed -e 's| ||g'` score_p=`printf '%2i' $t_pass`":$t_tot" rm -f tmploc #-- count Nb of different lines - ndf=`wc -l tmpdf | awk '{print $1}'` + ndf=`wc -l $TMP.dif | awk '{print $1}'` ndf=`expr $ndf / 2` - nn=`grep -c '^---' tmpdf` + nn=`grep -c '^---' $TMP.dif` ndf=`expr $ndf - $nn` if [ $ndf -le $NBLDIFF ] ; then - cat tmpdf + cat $TMP.dif else echo " $ndf lines differ ( $score_s , $score_p )" fi echo "< "`head -1 $outs`" ( $score_s )" >> $OUTPFIL echo "> "`head -1 $outp`" ( $score_p )" >> $OUTPFIL - cat tmpdf >> $OUTPFIL + cat $TMP.dif >> $OUTPFIL if [ $cdif -eq 0 ] ; then clin=`printf '%3i (%5s,%5s)' $ndf $score_s $score_p` else @@ -518,7 +566,7 @@ else echo " $type , of='$optf'" | tee -a $OUTPFIL fi - rm -f tmpfs tmpfp tmpdf + rm -f $TMP.fs $TMP.fp $TMP.dif fi done if [ $cmis -gt 0 ] ; then tmis='(-'$cmis')' ; else tmis='(--)' ; fi @@ -531,13 +579,18 @@ done echo "Start time: $sTime" | tee -a $OUTPFIL -echo "End time: "`date` | tee -a $OUTPFIL | tee -a $OUTPSUM +echo "End time: "`date` | tee -a $OUTPFIL | tee -a $OUTPSUM -rm -f ./dir_all ./slist ./plist +rm -f $TMP.dir_all $TMP.slist $TMP.plist if test $OUTPFIL = "TTT.$$" ; then rm -f $OUTPFIL ; fi if test "x$ADDRESS" != 'xnone' ; then - echo ".. send $OUTPSUM to $ADDRESS" - mail -s $OUTPSUM $ADDRESS < $OUTPSUM + #- change default sender address ("From:") using "mail -r" option + # since, by default, baudelaire uses USER@localhost.localdomain + #fullNam=`hostname -A | awk '{print $1}'` + #MAILCMD="$MAILCMD -r ${USER}@${fullNam}" + #--- + echo ".. send $OUTPSUM to '$ADDRESS' using '$MAILCMD'" + $MAILCMD -s 'check_outp summary' $ADDRESS < $OUTPSUM fi echo ".. cat $OUTPSUM" cat $OUTPSUM