/[MITgcm]/MITgcm_contrib/llc_hires/llc_4320/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_4320/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.2 by dimitri, Tue Oct 29 04:24:17 2013 UTC revision 1.12 by dimitri, Mon Feb 10 15:34:38 2014 UTC
# Line 1  Line 1 
1  For interactive session, Ivy Bridge nodes:  For interactive session, Ivy Bridge nodes:
2  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
3  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email
4  qsub -I -q long -l select=300:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
5  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00  
6    #############################
7  For batch submission:  # 90x90x19023 configuration
8  qsub -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 runscript  
9  qsub qsub_llc2160.csh  qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
10    module purge
11    module load  comp-intel/2012.0.032 netcdf/4.0
12    module use -a ~kjtaylor/modulefiles
13    module load sles11sp3/mpt-2.10-nasa201311271217
14    cd ~/llc_4320
15    cvs co MITgcm_code
16    cvs co MITgcm_contrib/llc_hires/llc_4320
17    cd MITgcm
18    mkdir build run
19    lfs setstripe -c -1 run
20    cd build
21    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
22    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
23    emacs readtile_mpiio.c
24        tileSizeX = 90;
25        tileSizeY = 90;
26    ../tools/genmake2 -of \
27     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
28     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
29    make depend
30    make -j 16
31    
32    cd ~/llc_4320/MITgcm/run
33    cp ../build/mitgcmuv mitgcmuv_90x90x19023
34    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
35    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
36    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
37    mv data.exch2_90x90x19023 data.exch2
38    emacs data
39    
40    export MPI_BUFS_PER_PROC=1024
41    export MPI_REQUEST_MAX=65536
42    export MPI_GROUP_MAX=1024
43    export MPI_NUM_MEMORY_REGIONS=8
44    export MPI_UNBUFFERED_STDIO=1
45    export MPI_MEMMAP_OFF=1
46    export MPI_UD_TIMEOUT=100
47    mpiexec -n 20400 ./mitgcmuv_90x90x19023
48    
49    tail -f STDOUT.00000 | grep advcfl_W
50    
51    #############################
52    # generate 60x60 blank tiles
53    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
54    module purge
55    module load  comp-intel/2012.0.032 netcdf/4.0
56    module use -a ~kjtaylor/modulefiles
57    module load sles11sp3/mpt-2.10-nasa201311271217
58    cd ~/llc_4320/MITgcm
59    mkdir run_60x60
60    lfs setstripe -c -1 run_60x60
61    cd build
62    rm *
63    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
64    emacs readtile_mpiio.c
65        tileSizeX = 60;
66        tileSizeY = 60;
67    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
68    emacs SIZE.h
69         &           sNx =  60,
70         &           sNy =  60,
71         &           nSx =   6,
72         &           nPx = 11232,
73         &           Nr  =  2 )
74    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
75    emacs eeboot_minimal.F
76    C         standardMessageUnit=errorMessageUnit
77             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
78             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
79    ../tools/genmake2 -of \
80     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
81     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
82    make depend
83    make -j 16
84    cd ~/llc_4320/MITgcm/run_60x60
85    cp ../build/mitgcmuv .
86    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
87    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
88    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
89    mv data.exch2_120x120x10901 data.exch2
90    emacs data
91     tRef =  18.89, 18.89,
92     sRef =  34.84, 34.84,
93     endtime=20.,
94     delR =   1.00,    1.14,
95    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
96    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
97    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
98    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
99    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
100    emacs data.exch2
101    # remove the blank tile list
102    export MPI_BUFS_PER_PROC=1024
103    export MPI_REQUEST_MAX=65536
104    export MPI_GROUP_MAX=1024
105    export MPI_NUM_MEMORY_REGIONS=8
106    export MPI_UNBUFFERED_STDIO=1
107    export MPI_MEMMAP_OFF=1
108    export MPI_UD_TIMEOUT=100
109    mpiexec -n 12000 ./mitgcmuv
110    
111    #############################
112    # generate 45x45 blank tiles
113    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
114    module purge
115    module load  comp-intel/2012.0.032 netcdf/4.0
116    module use -a ~kjtaylor/modulefiles
117    module load sles11sp3/mpt-2.10-nasa201311271217
118    cd ~/llc_4320/MITgcm
119    mkdir run_45x45
120    lfs setstripe -c -1 run_45x45
121    cd build
122    rm *
123    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
124    emacs readtile_mpiio.c
125        tileSizeX = 45;
126        tileSizeY = 45;
127    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
128    emacs SIZE.h
129         &           sNx =  45,
130         &           sNy =  45,
131         &           nSx =  12,
132         &           nPx = 9984,
133         &           Nr  =  2 )
134    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
135    emacs eeboot_minimal.F
136    C         standardMessageUnit=errorMessageUnit
137             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
138             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
139    ../tools/genmake2 -of \
140     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
141     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
142    make depend
143    make -j 16
144    cd ~/llc_4320/MITgcm/run_45x45
145    cp ../build/mitgcmuv .
146    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
147    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
148    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
149    mv data.exch2_120x120x10901 data.exch2
150    emacs data
151     tRef =  18.89, 18.89,
152     sRef =  34.84, 34.84,
153     endtime=20.,
154     delR =   1.00,    1.14,
155    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
156    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
157    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
158    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
159    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
160    emacs data.exch2
161    # remove the blank tile list
162    export MPI_BUFS_PER_PROC=1024
163    export MPI_REQUEST_MAX=65536
164    export MPI_GROUP_MAX=1024
165    export MPI_NUM_MEMORY_REGIONS=8
166    export MPI_UNBUFFERED_STDIO=1
167    export MPI_MEMMAP_OFF=1
168    export MPI_UD_TIMEOUT=100
169    mpiexec -n 12000 ./mitgcmuv
170    
171    #############################
172    # generate 48x48 blank tiles
173    qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com
174    module purge
175    module load  comp-intel/2012.0.032 netcdf/4.0
176    module use -a ~kjtaylor/modulefiles
177    module load sles11sp3/mpt-2.10-nasa201311271217
178    cd ~/llc_4320/MITgcm
179    mkdir run_48x48
180    lfs setstripe -c -1 run_48x48
181    cd build
182    rm *
183    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
184    emacs readtile_mpiio.c
185        tileSizeX = 48;
186        tileSizeY = 48;
187    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
188    emacs SIZE.h
189         &           sNx =  48,
190         &           sNy =  48,
191         &           nSx =  10,
192         &           nPx = 10530,
193         &           Nr  =  2 )
194    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
195    emacs eeboot_minimal.F
196    C         standardMessageUnit=errorMessageUnit
197             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
198             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
199    ../tools/genmake2 -of \
200     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
201     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
202    make depend
203    make -j 16
204    cd ~/llc_4320/MITgcm/run_48x48
205    cp ../build/mitgcmuv .
206    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
207    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
208    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
209    mv data.exch2_120x120x10901 data.exch2
210    emacs data.exch2
211    # remove blankList
212    emacs data
213     tRef =  18.89, 18.89,
214     sRef =  34.84, 34.84,
215     endtime=20.,
216     delR =   1.00,    1.14,
217    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
218    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
219    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
220    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
221    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
222    export MPI_BUFS_PER_PROC=1024
223    export MPI_REQUEST_MAX=65536
224    export MPI_GROUP_MAX=1024
225    export MPI_NUM_MEMORY_REGIONS=8
226    export MPI_UNBUFFERED_STDIO=1
227    export MPI_MEMMAP_OFF=1
228    export MPI_UD_TIMEOUT=100
229    mpiexec -n 12000 ./mitgcmuv
230    
231    #############################
232    # 120x120x10901 configuration
233    
234    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
235    module purge
236    module load  comp-intel/2012.0.032 netcdf/4.0
237    module use -a ~kjtaylor/modulefiles
238    module load sles11sp3/mpt-2.10-nasa201311271217
239    cd ~/llc_4320
240    cvs co MITgcm_code
241    cvs co MITgcm_contrib/llc_hires/llc_4320
242    cd MITgcm
243    mkdir build run
244    lfs setstripe -c -1 run
245    cd build
246    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
247    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
248    emacs readtile_mpiio.c
249        tileSizeX = 120;
250        tileSizeY = 120;
251    ../tools/genmake2 -of \
252     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
253     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
254    make depend
255    make -j 16
256    
257    cd ~/llc_4320/MITgcm/run
258    cp ../build/mitgcmuv mitgcmuv_120x120x10901
259    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
260    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
261    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
262    mv data.exch2_120x120x10901 data.exch2
263    emacs data
264    
265    export MPI_BUFS_PER_PROC=1024
266    export MPI_REQUEST_MAX=65536
267    export MPI_GROUP_MAX=1024
268    export MPI_NUM_MEMORY_REGIONS=8
269    export MPI_UNBUFFERED_STDIO=1
270    export MPI_MEMMAP_OFF=1
271    export MPI_UD_TIMEOUT=100
272    mpiexec -n 12000 ./mitgcmuv_120x120x10901
273    
274    tail -f STDOUT.00000 | grep advcfl_W
275    
276    ==============
277    
278    qsub -I -q R3089666 -l select=1750:model=ivy:aoe=sles11,walltime=04:00:00
279    tcsh
280    cd ~/llc_4320/MITgcm
281    mkdir run
282    lfs setstripe -c -1 run
283    cd run
284    cat $PBS_NODEFILE | awk '{for (i=0;i<20;++i) print $0}' > mynodes
285    setenv PBS_NODEFILE mynodes
286    cp /nobackupp8/chenze/run/mitgcmuv_72x72x29297 .
287    ln -sf /nobackupp8/chenze/run/pickup_seaice_0000000360.meta pickup_seaice.0000000180.meta
288    ln -sf /nobackupp8/chenze/run/pickup_seaice_0000000360.data pickup_seaice.0000000180.data
289    ln -sf /nobackupp8/chenze/run/pickup_0000000360.meta pickup.0000000180.meta
290    ln -sf /nobackupp8/chenze/run/pickup_0000000360.data pickup.0000000180.data
291    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
292    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
293    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
294    mv data.exch2_72x72x29297 data.exch2
295    module purge
296    module load  comp-intel/2012.0.032 netcdf/4.0
297    module use -a ~kjtaylor/modulefiles
298    module load sles11sp3/mpt-2.10-nasa201311271217
299    setenv MPI_BUFS_PER_PROC 512
300    setenv MPI_REQUEST_MAX 65536
301    setenv MPI_GROUP_MAX 1024
302    setenv MPI_NUM_MEMORY_REGIONS 8
303    setenv MPI_UNBUFFERED_STDIO 1
304    setenv MPI_MEMMAP_OFF 1
305    export MPI_UD_TIMEOUT=100
306    
307    mpiexec -n 35000 ./mitgcmuv_72x72x29297
308    
309    tail -f STDOUT.00000 | grep advcfl_w
310    
311  ==============  ==============
312    
313    cd ~/llc_4320
314  cvs co MITgcm_code  cvs co MITgcm_code
315    cvs co MITgcm_contrib/llc_hires/llc_4320
316    # set correct tileSizeX and tileSizeY in MITgcm_contrib/llc_hires/llc_4320/cpde-asyn/readtile_mpiio.c
317  cd MITgcm  cd MITgcm
318  module purge  module purge
319  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.06r6 netcdf/4.0  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
320  mkdir build run  mkdir build run
321  lfs setstripe -c -1 /nobackupp5/dmenemen/llc_4320/MITgcm/run  lfs setstripe -c -1 run
322  cd build  cd build
323  cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_180x180x5015 SIZE.h  cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_72x72x29297 SIZE.h
324  ../tools/genmake2 -of ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \  ../tools/genmake2 -of \
325     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
326   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
327  make depend  make depend
328  make -j 16  make -j 16
329  cd ../run  cd ../run
330  ln -sf ../build/mitgcmuv .  cp ../build/mitgcmuv mitgcmuv_72x72x29297
331    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
332    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
333    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
334    mv data.exch2_72x72x29297 data.exch2
335    export MPI_NUM_MEMORY_REGIONS=256
336    mpiexec -n 35000 ./mitgcmuv_72x72x29297
337    
338    tail -f STDOUT.00000 | grep advcfl_w
339    
340    ==============
341    
342    cd ~/llc_4320
343    cvs co MITgcm_code
344    cvs co MITgcm_contrib/llc_hires/llc_4320
345    # set correct tileSizeX and tileSizeY in MITgcm_contrib/llc_hires/llc_4320/cpde-asyn/readtile_mpiio.c
346    cd MITgcm
347    module purge
348    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
349    mkdir build run_180x180x5015
350    lfs setstripe -c -1 run_180x180x5015
351    cd build
352    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_180x180x5015 SIZE.h
353    ../tools/genmake2 -of \
354     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
355     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
356    make depend
357    make -j 16
358    cd ../run_180x180x5015
359    cp ../build/mitgcmuv mitgcmuv_180x180x5015
360  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
361  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
362  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
363  mpiexec -n 6000 ./mitgcmuv  mv data.exch2_72x72x29297 data.exch2
364    export MPI_NUM_MEMORY_REGIONS=256
365    mpiexec -n 6000 ./mitgcmuv_180x180x5015
366    
367  ==============  ==============
368    
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.12
  ViewVC Help
Powered by ViewVC 1.1.22