/[MITgcm]/MITgcm_contrib/jmc_script/run_cpl_test.aces
ViewVC logotype

Annotation of /MITgcm_contrib/jmc_script/run_cpl_test.aces

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.1 - (hide annotations) (download)
Mon Dec 6 02:56:35 2010 UTC (14 years, 7 months ago) by jmc
Branch: MAIN
local version for ACES and darwin (beagle) cluster of a simple
 script to compile and run coupled verification set-up.

1 jmc 1.1 #! /bin/sh
2    
3     # $Header: /u/gcmpack/MITgcm/verification/cpl_aim+ocn/run_cpl_test_itrda,v 1.13 2009/12/26 01:58:17 jmc Exp $
4     # $Name: $
5    
6     #- default:
7     Npr=3 ; NpOc=1 ;
8     #Npr=25; NpOc=12;
9     MTH=
10     MTHo=
11     MTHa=
12     GMKopt='-ieee'
13     #GMKopt=
14     rnkO=1 ; rnkA=`expr $rnkO + $NpOc`
15    
16     #- parse options:
17     if [ $# -ge 1 ] ; then if test $1 = '-mth' ; then
18     MTH='-omp' ; shift
19     if test -f input_ocn/eedata.mth ; then MTHo=$MTH ; fi
20     if test -f input_atm/eedata.mth ; then MTHa=$MTH ; fi
21     fi ; fi
22     if [ $# -ne 1 ]
23     then
24     echo 'Usage:'`basename $0`' [opt] step'
25     echo ' => test coupled set-up on ACES cluster'
26     echo 'opt = -mth : compile and run (if eedata.mth) 2-threads for ocn & atm'
27     echo ' step = 0 : clean all directories'
28     echo ' step = 1 : compile the 3 executables (cpl,ocn,atm)'
29     echo ' step = 2 : copy input files and dir(s)'
30     echo " step = 3 : run with $Npr mpi processes"
31     echo ' step = 4 : check the results'
32     echo ' step = 5 : remove output files in rank_0,1,2 dir.'
33     exit
34     fi
35     kpr=$1
36     dir=`pwd`
37    
38     #============================================================================
39    
40     #- only works from a 3 (or more) nodes PBS bach session on ACES cluster
41     # (tested on ao) ( qsub -I -l nodes=3 -or-
42     # qsub -I -l nodes=3:ppn=2 )
43     # need to be on one computer node,
44     # and with the correct module loaded (i.e, one of the 3):
45     # > module load mpich/gnu
46     # > module load mpich/intel netcdf/3.6.1/icc
47     # export MPI_INC_DIR='/usr/local/pkg/mpich/mpich-intel/include/'
48     # > module load mpich/pgi
49     if test 'dummy'$PBS_NODEFILE = 'dummy'
50     then
51     echo ' $PBS_NODEFILE not defined'
52     echo "for now, works only from a (at least) $Npr nodes PBS bach session"
53     exit 8
54     else nprc=`wc -l $PBS_NODEFILE | awk '{print $1}'`
55     if [ $nprc -lt $Npr ] ; then
56     echo ' Nb of nodes from $PBS_NODEFILE =' $nprc
57     echo "for now, works only from a (at least) $Npr nodes PBS bach session"
58     exit 8
59     fi
60     fi
61    
62     #============================================================================
63    
64     if test $kpr = 0
65     then
66     rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
67     rm -f build_???/TTT.*make.* build_???/TTT.mkdepend.*
68     /bin/rm -r -f rank_? rank_1? rank_2?
69     if test -f build_cpl/Makefile ; then cd build_cpl ; make Clean ; cd .. ; fi
70     if test -f build_ocn/Makefile ; then cd build_ocn ; make Clean ; cd .. ; fi
71     if test -f build_atm/Makefile ; then cd build_atm ; make Clean ; cd .. ; fi
72     fi
73     if test $kpr = 5
74     then
75     echo 'remove output files in rank_0,1,2 dir.'
76     rm -f pr_group std_outp comp_res.{ocn,atm,land,sice}
77     if test -d rank_0 ; then cd rank_0 ; rm -f Coupler.0000.clog ; cd .. ; fi
78     if test -d rank_$rnkO
79     then cd rank_$rnkO ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
80     if test -d rank_$rnkA
81     then cd rank_$rnkA ; rm -f *.data *.meta STD???.0000 UV-*.0001.clog ; cd .. ; fi
82     fi
83    
84     if test $kpr = 1
85     then
86    
87     #- find the optfile that corresponds to the currently loaded mpich module
88     . /etc/profile.d/modules.sh
89     module list -t >& tmp_compiler
90     compiler=`sed -n 's/mpich.*\///p' tmp_compiler`
91     case $compiler in
92     'gnu') OPTFILE="../../tools/build_options/linux_ia32_g77+mpi_aces" ;;
93     'intel') OPTFILE="../../tools/build_options/linux_ia32_ifort+mpi_aces" ;;
94     'pgi') OPTFILE="../../tools/build_options/linux_ia32_pgf77+mpi_aces" ;;
95     *) echo 'ERROR: cannot use compiler:' $compiler ;
96     echo ""; cat tmp_compiler ; exit 7 ;;
97     esac
98     rm -f tmp_compiler
99     echo " Using optfile: $OPTFILE (compiler=$compiler) $MTH"
100     zz=`echo $OPTFILE | grep -c '^\/'`
101     if test $zz = 0 ; then OPTFILE="../$OPTFILE" ; fi
102     #---
103    
104     echo '==== compile coupler:'
105     cd build_cpl
106     echo ' --- genmake2 (cpl):'
107     ../../../tools/genmake2 -of $OPTFILE -mpi $GMKopt > TTT.genmake.$$
108     tail -5 TTT.genmake.$$
109     echo ' --- make depend (cpl):'
110     make depend > TTT.mkdepend.$$
111     tail -5 TTT.mkdepend.$$
112     echo ' --- make (cpl):'
113     make > TTT.make.$$ 2>&1
114     tail -10 TTT.make.$$
115     echo ' ' ; cd $dir
116    
117     echo '==== compile OGCM:'
118     cd build_ocn
119     echo ' --- genmake2 (ocn):'
120     ../../../tools/genmake2 -of $OPTFILE -mpi $MTHo $GMKopt > TTT.genmake.$$
121     tail -5 TTT.genmake.$$
122     echo ' --- make depend (ocn):'
123     make depend > TTT.mkdepend.$$
124     tail -10 TTT.mkdepend.$$
125     echo ' --- make (ocn):'
126     make > TTT.make.$$ 2>&1
127     tail -10 TTT.make.$$
128     echo ' ' ; cd $dir
129    
130     echo '==== compile AGCM:'
131     cd build_atm
132     echo ' --- genmake2 (atm):'
133     ../../../tools/genmake2 -of $OPTFILE -mpi $MTHa $GMKopt > TTT.genmake.$$
134     tail -5 TTT.genmake.$$
135     echo ' --- make depend (atm):'
136     make depend > TTT.mkdepend.$$
137     tail -10 TTT.mkdepend.$$
138     echo ' --- make (atm):'
139     make > TTT.make.$$ 2>&1
140     tail -10 TTT.make.$$
141     echo ' ' ; cd $dir
142    
143     ls -l build_???/mitgcmuv
144    
145     fi
146    
147     if test $kpr = 2
148     then
149     echo 'rm dir:' rank_? rank_1? rank_2?
150     /bin/rm -r -f rank_? rank_1? rank_2?
151     echo 'Link files from dir:' input_cpl '->' rank_0
152     mkdir rank_0
153     ( cd rank_0 ; ln -s ../input_cpl/* . )
154    
155     n=$rnkO
156     echo 'Link files from dir:' input_ocn '->' rank_$n
157     mkdir rank_$n
158     cd rank_$n
159     ln -s ../input_ocn/* .
160     if test -x prepare_run ; then ./prepare_run ; fi
161     if test "x$MTHo" != x ; then
162     echo " MTH run: mv -f eedata.mth eedata"
163     if test -h eedata ; then rm -f eedata ; fi
164     mv -f eedata.mth eedata
165     fi
166     cd $dir
167     n=`expr $n + 1`
168     while [ $n -le $NpOc ] ; do
169     ln -s rank_$rnkO rank_$n
170     n=`expr $n + 1`
171     done
172    
173     n=$rnkA
174     echo 'Link files from dir:' input_atm '->' rank_$n
175     mkdir rank_$n
176     cd rank_$n
177     ln -s ../input_atm/* .
178     if test -x prepare_run ; then ./prepare_run ; fi
179     if test "x$MTHa" != x ; then
180     echo " MTH run: mv -f eedata.mth eedata"
181     if test -h eedata ; then rm -f eedata ; fi
182     mv -f eedata.mth eedata
183     fi
184     cd $dir
185     n=`expr $n + 1`
186     while [ $n -lt $Npr ] ; do
187     ln -s rank_$rnkA rank_$n
188     n=`expr $n + 1`
189     done
190    
191     fi
192    
193     if test $kpr = 3
194     then
195     ROOTDIR=$dir
196     # rm -f rank_?/pickup*.ckptA.00?.00?.??ta
197     echo $ROOTDIR
198     tmpfil=TTT.$$
199    
200     #--- running on the same node:
201     list='' ; nc=0; xx=`hostname`
202     while [ $nc -lt $Npr ] ; do list="$list $xx" ; nc=`expr $nc + 1` ; done
203     #-- On darwin cluster node (from qrsh session):
204     #JOB_ID=`qstat | sed -n '3,$ p' | grep " $USER " | awk '{print $1}'`
205     #NODEFILE="/tmp/$JOB_ID.1.darwin/machines"
206     #echo " JOB_ID = '$JOB_ID' ; NODEFILE = '$NODEFILE'"
207     #-- On ACES cluster (in PBS batch job):
208     NODEFILE=$PBS_NODEFILE
209     #--- running on different nodes:
210     ls -l $NODEFILE
211     nprc=`cat $NODEFILE | uniq | wc -l`
212     if [ $nprc -ge $Npr ] ; then
213     list=`cat $NODEFILE | uniq | head -$Npr`
214     else
215     list=`cat $NODEFILE | head -$Npr`
216     fi
217    
218     nc=0; nn=0; dd1=cpl ;
219     rm -f pr_group ; touch pr_group
220     for xx in $list
221     do
222     echo $xx $nn $dir/build_$dd1/mitgcmuv >> pr_group
223     nc=`expr $nc + 1`
224     if [ $nc -le $NpOc ] ; then dd1=ocn ; else dd1=atm ; fi
225     nn=1
226     done
227    
228     cd $ROOTDIR
229     if test "x$MTH" != x ; then
230     export OMP_NUM_THREADS=2 ; export KMP_STACKSIZE=400m
231     if test "x$MTHo" != x ; then
232     echo -n " run OCN ($MTHo) with $OMP_NUM_THREADS threads ;"
233     fi
234     if test "x$MTHa" != x ; then
235     echo -n " run ATM ($MTHa) with $OMP_NUM_THREADS threads ;"
236     fi
237     echo ""
238     fi
239     # /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -pg pr_group -wd $ROOTDIR --gm-kill 5 -v ./build_cpl/mitgcmuv > std_outp 2>&1
240     #- with mpich-1 (on danton, ACES):
241     mpirun -p4pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
242     #- with mpich-mx (on beagle):
243     # mpirun -pg pr_group -v ./build_cpl/mitgcmuv > std_outp 2>&1
244     tail -20 std_outp
245     ls -l rank_$rnkO/pickup*.ckptA.001.001.data
246     ls -l rank_$rnkA/pickup*.ckptA.001.001.data
247    
248     fi
249    
250     if test $kpr = 4
251     then
252    
253     if test -f rank_$rnkO/STDOUT.0000
254     then echo '==> check Ocean output:'
255     /home/jmc/bin/comp_res rank_$rnkO/STDOUT.0000 results/ocnSTDOUT.0000
256     mv -f comp_res.log comp_res.ocn
257     echo ' '
258     else echo "No Ocean output file in rank_$rnkO" ; fi
259    
260     if test -f rank_$rnkA/STDOUT.0000
261     then
262     echo '==> check Atmos output:'
263     /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000
264     mv -f comp_res.log comp_res.atm
265     echo '==> check Land output:'
266     /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 L
267     mv -f comp_res.log comp_res.land
268     echo '==> check thSIce output:'
269     /home/jmc/bin/comp_res rank_$rnkA/STDOUT.0000 results/atmSTDOUT.0000 I
270     mv -f comp_res.log comp_res.sice
271     echo ' '
272     else echo "No Atmos output file in rank_$rnkA" ; fi
273    
274     fi
275    
276     exit 0

  ViewVC Help
Powered by ViewVC 1.1.22