--- mitgcm.org/scripts/check_outp	2009/04/06 18:12:34	1.6
+++ mitgcm.org/scripts/check_outp	2010/03/22 14:45:20	1.21
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.6 2009/04/06 18:12:34 jmc Exp $
+# $Header: /home/ubuntu/mnt/e9_copy/mitgcm.org/scripts/check_outp,v 1.21 2010/03/22 14:45:20 jmc Exp $
 #
 #  The purpose of this script is to compare
 #  the latest output with the previous one (from the same platform with
@@ -32,6 +32,95 @@
     exit 1
 }
 
+#  create list of results dir for one platform:
+make_resdir_list()
+{
+      if test $monthDir == 1 ; then
+	dir_list=`grep $mname ./dir_all`
+      else
+	dir_list=`( cd $INDIR ; ls -1 -t ../$PREV_P/*${mname}*/summary.txt 2> /dev/null | sed 's/\/summary.txt//' )`
+	#echo "dir_list='$dir_list'"
+      fi
+	for i in $dir_list ; do
+
+	    dir=$INDIR"/"$i
+	    OPTFILE=
+	    if test -r $dir/summary.txt ; then
+		comm=`grep 'OPTFILE=' $dir/summary.txt`
+		eval $comm
+		OPTFILE=${OPTFILE##*/}
+	    fi
+	    if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then
+		comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null`
+		eval $comm
+		OPTFILE=${OPTFILE##*/}
+	    fi
+	    if test "x$OPTFILE" = x ; then
+		comm=`grep '^# OPTFILE=' $dir/*/Makefile* 2>/dev/null | head -1`
+		comm=${comm##*#}
+		eval $comm
+		OPTFILE=${OPTFILE##*/}
+	    fi
+	    if test "x$OPTFILE" = x ; then
+		OPTFILE="not_explicitly_specified"
+	    fi
+
+	    ADJOINT=
+	    RESTART=0
+	    NOI3E=0
+	    MTH=0
+	    if test -r $dir/summary.txt ; then
+		comm=`grep 'ADJOINT=true' $dir/summary.txt 2>/dev/null`
+		eval $comm
+		RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt`
+		NOI3E=`grep -c "^run: .*testreport.* '*-noieee'*" $dir/summary.txt`
+		MTH=`grep -c "^run: .*testreport.* -mth" $dir/summary.txt`
+	    fi
+	    if test "x$RESTART" = x0 ; then
+		kind="forward"
+		test "x$ADJOINT" = xtrue  &&  kind="adjoint"
+	    else
+		kind="restart"
+	    fi
+	    if test "x$MTH" = x1 ; then
+		yy=`echo $OPTFILE | grep -c '+mth$'`
+		if test $yy = 0 ; then OPTFILE="${OPTFILE}+mth" ; fi
+	    fi
+	    if test "x$NOI3E" = x1 ; then
+		OPTFILE="${OPTFILE}.noieee"
+	    fi
+  	    day=`echo $i   | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'`
+  	    ttt=`echo $day | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'`
+  	    day=`echo $ttt | sed -e 's|_| |g' |awk '{print $NF}'`
+
+            if [ $day -lt $FIRSTDAY ] ; then
+		echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./plist
+            elif [ $day -le $LASTDAY ] ; then
+		echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./slist
+	    else
+		if test $dBug = 't' ; then
+		   echo "discard day='$day' from: $i" | tee -a $OUTPFIL
+		fi
+	    fi
+
+	done
+	if test $dBug = 't' ; then
+	  if test $monthDir == 1 ; then
+	    echo "---- current list (n=$monthDir) :" | tee -a $OUTPFIL
+		cat ./slist | tee -a $OUTPFIL
+	  fi
+          num1=`wc -l ./slist | awk '{print $1}'`
+	  if test $num1 -gt 0 ; then
+	  echo   ".... previous list (n=$monthDir):" | tee -a $OUTPFIL
+		cat ./plist | tee -a $OUTPFIL | head -20
+	  fi
+          echo "----" | tee -a $OUTPFIL
+	fi
+}
+
+#--------------------------------------------------------------------------
+# here starts the sequential part of the script:
+
 #CURR_PER=`date +%Y`"_"`date +%m`
 CURR_DAY=`date +%Y%m%d`
 # defaults
@@ -43,6 +132,8 @@
 ADDRESS='none'
 dBug=f
 sTime=`date`
+#- to get case insensitive "ls" (and order of tested experiments)
+export LC_ALL="en_US.UTF-8"
 
 #  Parse options
 ac_prev=
@@ -176,9 +267,9 @@
 ( cd $INDIR ; ls -1 -t */summary.txt | sed 's/\/summary.txt//' ) > ./dir_all
 
 if test "x$MACHINES" = "x_All_" ; then
-  MACHINES="faulks aces eddy bay meander lagoon hugo"
-  MACHINES="$MACHINES columbia edvir rays solasrv sx8 xd1"
-  MACHINES="$MACHINES beagle batsi starp"
+  MACHINES="faulks meander lagoon aces dickens danton beagle harbor"
+  MACHINES="$MACHINES columbia pleiades iblade rays solssrv solasrv sx8"
+  MACHINES="$MACHINES trane dodongo dokdo bigred"
 
   MALL=`cat ./dir_all | sed -e 's|_| |g' | awk '{print $2}' | sort | uniq`
   for madd in $MALL ; do
@@ -203,89 +294,9 @@
     #     results from previous period in "plist"
     rm -f ./plist ./slist ; touch ./plist ./slist
 
-    for n in $listNum ; do
-      num1=`wc -l ./slist | awk '{print $1}'`
-      num0=`wc -l ./plist | awk '{print $1}'`
-      if test $n == 2 ; then
-	num2=`expr $num1 \* $n`
-	if [ $num0 -lt $num2 ] ; then
-	  dir_list=`( cd $INDIR ; ls -1 -t ../$PREV_P/*${mname}*/summary.txt 2> /dev/null | sed 's/\/summary.txt//' )`
-          #echo "dir_list='$dir_list'"
-	fi
-      else
-	num2=$n
-	dir_list=`grep $mname ./dir_all`
-      fi
-      if [ $num0 -lt $num2 ] ; then
-	for i in $dir_list ; do
-
-	    dir=$INDIR"/"$i
-	    OPTFILE=
-	    if test -r $dir/summary.txt ; then
-		comm=`grep 'OPTFILE=' $dir/summary.txt`
-		eval $comm
-		OPTFILE=${OPTFILE##*/}
-	    fi
-	    if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then
-		comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null`
-		eval $comm
-		OPTFILE=${OPTFILE##*/}
-	    fi
-	    if test "x$OPTFILE" = x ; then
-		comm=`grep '^# OPTFILE=' $dir/*/Makefile* 2>/dev/null | head -1`
-		comm=${comm##*#}
-		eval $comm
-		OPTFILE=${OPTFILE##*/}
-	    fi
-	    if test "x$OPTFILE" = x ; then
-		OPTFILE="not_explicitly_specified"
-	    fi
-
-	    ADJOINT=
-	    RESTART=0
-	    NOI3E=0
-	    if test -r $dir/summary.txt ; then
-		comm=`grep 'ADJOINT=true' $dir/summary.txt 2>/dev/null`
-		eval $comm
-		RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt`
-		NOI3E=`grep -c "^run: .*testreport .* '*-noieee'* " $dir/summary.txt`
-	    fi
-	    if test "x$RESTART" = x0 ; then
-		kind="forward"
-		test "x$ADJOINT" = xtrue  &&  kind="adjoint"
-	    else
-		kind="restart"
-	    fi
-	    if test "x$NOI3E" = x1 ; then
-		OPTFILE="${OPTFILE}.noieee"
-	    fi
-  	    day=`echo $i   | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'`
-  	    ttt=`echo $day | sed -e 's/_[0-9]$//' | sed -e 's/_[0-9][0-9]$//'`
-  	    day=`echo $ttt | sed -e 's|_| |g' |awk '{print $NF}'`
-
-            if [ $day -lt $FIRSTDAY ] ; then
-		echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./plist
-            elif [ $day -le $LASTDAY ] ; then
-		echo "$OPTFILE$kind $day $OPTFILE $kind $i" >> ./slist
-	    else
-		if test $dBug = 't' ; then
-		   echo "discard day='$day' from: $i" | tee -a $OUTPFIL
-		fi
-	    fi
-
-	done
-	if test $dBug = 't' ; then
-	  if test $n == 1 ; then
-	    echo "---- current list (n=$n) :" | tee -a $OUTPFIL
-		cat ./slist | tee -a $OUTPFIL
-	  fi
-	  echo   ".... previous list (n=$n):" | tee -a $OUTPFIL
-		cat ./plist | tee -a $OUTPFIL
-	fi
-
-      fi
-    done
-    if test $dBug = 't' ; then echo "----" | tee -a $OUTPFIL ; fi
+    #- create list of results (slist & plist) from current month directory
+    monthDir=1
+    make_resdir_list
 
     #  Do we have any data?  If so, create the latest pointer.
     num=`wc -l ./slist | awk '{print $1}'`
@@ -297,13 +308,30 @@
           ctot=`expr $ctot + 1`
 	  sline=`grep "^$key " ./slist | head -1`
  	  sdir=`echo $sline | cut -d " " -f 5`
-          kind=`echo $sline | cut -d " " -f 4`
+          type=`echo $sline | cut -d " " -f 4`
           optf=`echo $sline | cut -d " " -f 3`
           num=`grep -c "^$key" ./plist`
+          if test $num = 0 -a $monthDir = 1 ; then
+          #- add results from previous month directory to "plist".
+            monthDir=2
+            make_resdir_list
+            num=`grep -c "^$key" ./plist`
+          fi
 	  #- discard unsafe test:
-	  dd=`echo $optf | grep -c 'gfortran.*mth'`
-	  #if [ $dd -ge 1 ] ; then num=-1 ; fi
-          if test $num -gt 0 ; then
+	  dd=`echo $optf | grep -c 'gfortran+mth'`
+	  #if test $mname = 'lagoon' -a $dd = 1 ; then num=-1 ; fi
+	  #if test $mname = 'harbor' -a $dd = 1 ; then num=-1 ; fi
+	  #if test $mname = 'aces' -a $optf = 'linux_ia32_open64' ; then num=-1 ; fi
+	  #if test $mname = 'aces' -a $optf = 'linux_ia32_g95' ; then num=-1 ; fi
+	  dd=`echo $optf | grep -c 'bigred.*+mth'`
+	  #if test $mname = 'bigred' -a $dd = 1 ; then num=-1 ; fi
+	  if test $num -lt 0 ; then
+	    echo "  discard $sdir : $type , of='$optf'" | tee -a $OUTPFIL
+            cmis=`expr $cmis + 1`
+          elif test $num = '0' ; then
+            echo "no previous test for:" $key | tee -a $OUTPFIL
+            cmis=`expr $cmis + 1`
+	  else
 	    pline=`grep "^$key " ./plist | head -1`
 	    if test $dBug = 't' ; then 
 		echo "s='$sline'" | tee -a $OUTPFIL
@@ -318,11 +346,26 @@
  	    sed -n "5,$ p" $outp | sed '/^[YN] [YN] [YN] [YN]/ s/ \. //g' \
 				 | sed 's/ OPTFILE=.*\// OPTFILE=/g' \
                                  | grep -v '^[A-S][a-t]* time:  ' > tmpfp
+	    #-- re-order list of exp (according to local "sort" order)
+	    listD=`grep '^[YN] [YN] [YN] [YN]' tmpfs | awk '{print $NF}' | sort`
+	    for xx in $listD
+	    do
+		sed "/ $xx\$/d" tmpfs > tmpfn
+		sed -n "/ $xx\$/p" tmpfs >> tmpfn
+		mv tmpfn tmpfs
+	    done
+	    listD=`grep '^[YN] [YN] [YN] [YN]' tmpfp | awk '{print $NF}' | sort`
+	    for xx in $listD
+	    do
+		sed "/ $xx\$/d" tmpfp > tmpfn
+		sed -n "/ $xx\$/p" tmpfp >> tmpfn
+		mv tmpfn tmpfp
+	    done
 	    echo -n "=== diff $sdir $pdir :" | tee -a $OUTPFIL
 	    diff tmpfs tmpfp > tmpdf
 	    yy=$?
 	    if test $yy != '0' ; then
-                echo " $kind , of='$optf'" | tee -a $OUTPFIL
+                echo " $type , of='$optf'" | tee -a $OUTPFIL
 		#grep '^run: ' $outp
 		#grep '^run: ' $outs
 		#-- score for each test:
@@ -356,14 +399,9 @@
                 cdif=`expr $cdif + 1`
 		echo '----------------------------------------' | tee -a $OUTPFIL
 	    else
-		echo "" | tee -a $OUTPFIL
+		echo " $type , of='$optf'" | tee -a $OUTPFIL
 	    fi
 	    rm -f tmpfs tmpfp tmpdf
-	  elif test $num = '0' ; then
-            echo "no previous test for:" $key | tee -a $OUTPFIL
-            cmis=`expr $cmis + 1`
-	  else
-            cmis=`expr $cmis + 1`
           fi
 	done
         if [ $cmis -gt 0 ] ; then tmis='(-'$cmis')' ; else tmis='(--)' ; fi