/[MITgcm]/MITgcm_contrib/llc_hires/llc_4320/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_4320/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.8 by dimitri, Fri Jan 10 16:50:35 2014 UTC revision 1.16 by dimitri, Thu Apr 14 03:51:44 2016 UTC
# Line 1  Line 1 
1  For interactive session, Ivy Bridge nodes:  #############################
2  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email  # 90x90x19023 configuration
3  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email  module purge
4  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email  module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
5  qsub -I -q long -l select=300:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M email  cd ~/llc_4320
6    cvs co MITgcm_code
7    cvs co MITgcm_contrib/llc_hires/llc_4320
8    cd MITgcm
9    mkdir build run485568
10    lfs setstripe -c -1 run_485568
11    cd build
12    rm *
13    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
14    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
15    emacs readtile_mpiio.c
16        tileSizeX = 90;
17        tileSizeY = 90;
18    ../tools/genmake2 -of \
19     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
20     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
21    make depend
22    make -j 16
23    
24    cd ~/llc_4320/MITgcm/run_485568
25    cp ../build/mitgcmuv mitgcmuv_90x90x19023_intel.2015.0.090
26    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
27    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
28    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
29    emacs data
30     nIter0=485568,
31    ln -sf ../run/pickup_0000485568.data pickup.0000485568.data
32    ln -sf ../run/pickup_0000485568.meta pickup.0000485568.meta
33    ln -sf ../run/pickup_seaice_0000485568.data pickup_seaice.0000485568.data
34    ln -sf ../run/pickup_seaice_0000485568.meta pickup_seaice.0000485568.meta
35    
36    qsub -I -q long -l select=850:ncpus=24:model=has,walltime=120:00:00 -m abe -M menemenlis@me.com
37    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
38    cd ~/llc_4320/MITgcm/run_485568
39    cp data.exch2_90x90x19023 data.exch2
40    mpiexec -n 20400 ./mitgcmuv_90x90x19023_intel.2015.0.090
41    tail -f STDOUT.00000 | grep advcfl_W
42    
43    
44    #############################
45    # 120x120x10901 configuration
46    module purge
47    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
48    cd ~/llc_4320/MITgcm/build
49    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
50    emacs readtile_mpiio.c
51        tileSizeX = 120;
52        tileSizeY = 120;
53    make -j 16
54    cd ~/llc_4320/MITgcm/run_485568
55    cp ../build/mitgcmuv mitgcmuv_120x120x10901_intel.2015.0.090
56    
57    qsub -I -q testing_free -l select=428:ncpus=28:model=bro,walltime=8:00:00 -m abe -M menemenlis@me.com
58    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
59    cd ~/llc_4320/MITgcm/run_485568
60    cp data.exch2_120x120x10901 data.exch2
61    mpiexec -n 11984 ./mitgcmuv_120x120x10901_intel.2015.0.090
62    tail -f STDOUT.00000 | grep advcfl_W
63    
64    
65    #############################
66    # 180x180x5015 configuration
67    module purge
68    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
69    cd ~/llc_4320/MITgcm/build
70    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_180x180x5015 SIZE.h
71    emacs readtile_mpiio.c
72        tileSizeX = 180;
73        tileSizeY = 180;
74    make -j 16
75    cd ~/llc_4320/MITgcm/run_485568
76    cp ../build/mitgcmuv mitgcmuv_180x180x5015_intel.2015.0.090
77    
78    qsub -I -q testing_free -l select=214:ncpus=28:model=bro,walltime=8:00:00 -m abe -M menemenlis@me.com
79    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
80    cd ~/llc_4320/MITgcm/run_485568
81    cp data.exch2_180x180x5015 data.exch2
82    mpiexec -n 5992 ./mitgcmuv_180x180x5015_intel.2015.0.090
83    tail -f STDOUT.00000 | grep advcfl_W
84    
85    
86    #############################
87    # 48x48x64670 configuration
88    
89    module purge
90    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
91    cd ~/llc_4320
92    cvs co MITgcm_code
93    cvs co MITgcm_contrib/llc_hires/llc_4320
94    cd MITgcm
95    mkdir build run
96    lfs setstripe -c -1 run
97    cd build
98    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_48x48x64670 SIZE.h
99    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
100    emacs readtile_mpiio.c
101        tileSizeX = 48;
102        tileSizeY = 48;
103    ../tools/genmake2 -of \
104     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
105     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
106    make depend
107    make -j 16
108    
109    cd ~/llc_4320/MITgcm/run
110    cp ../build/mitgcmuv mitgcmuv_48x48x64670
111    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/data.exch2_90x90x19023 data.exch2
112    
113    
114    #############################
115    # generate 60x60 blank tiles
116    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
117    module purge
118    module load  comp-intel/2012.0.032 netcdf/4.0
119    module use -a ~kjtaylor/modulefiles
120    module load sles11sp3/mpt-2.10-nasa201311271217
121    cd ~/llc_4320/MITgcm
122    mkdir run_60x60
123    lfs setstripe -c -1 run_60x60
124    cd build
125    rm *
126    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
127    emacs readtile_mpiio.c
128        tileSizeX = 60;
129        tileSizeY = 60;
130    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
131    emacs SIZE.h
132         &           sNx =  60,
133         &           sNy =  60,
134         &           nSx =   6,
135         &           nPx = 11232,
136         &           Nr  =  2 )
137    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
138    emacs eeboot_minimal.F
139    C         standardMessageUnit=errorMessageUnit
140             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
141             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
142    ../tools/genmake2 -of \
143     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
144     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
145    make depend
146    make -j 16
147    cd ~/llc_4320/MITgcm/run_60x60
148    cp ../build/mitgcmuv .
149    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
150    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
151    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
152    mv data.exch2_120x120x10901 data.exch2
153    emacs data
154     tRef =  18.89, 18.89,
155     sRef =  34.84, 34.84,
156     endtime=20.,
157     delR =   1.00,    1.14,
158    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
159    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
160    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
161    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
162    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
163    emacs data.exch2
164    # remove the blank tile list
165    export MPI_BUFS_PER_PROC=1024
166    export MPI_REQUEST_MAX=65536
167    export MPI_GROUP_MAX=1024
168    export MPI_NUM_MEMORY_REGIONS=8
169    export MPI_UNBUFFERED_STDIO=1
170    export MPI_MEMMAP_OFF=1
171    export MPI_UD_TIMEOUT=100
172    mpiexec -n 12000 ./mitgcmuv
173    
174    
175    #############################
176    # generate 45x45 blank tiles
177    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
178    module purge
179    module load  comp-intel/2012.0.032 netcdf/4.0
180    module use -a ~kjtaylor/modulefiles
181    module load sles11sp3/mpt-2.10-nasa201311271217
182    cd ~/llc_4320/MITgcm
183    mkdir run_45x45
184    lfs setstripe -c -1 run_45x45
185    cd build
186    rm *
187    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
188    emacs readtile_mpiio.c
189        tileSizeX = 45;
190        tileSizeY = 45;
191    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
192    emacs SIZE.h
193         &           sNx =  45,
194         &           sNy =  45,
195         &           nSx =  12,
196         &           nPx = 9984,
197         &           Nr  =  2 )
198    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
199    emacs eeboot_minimal.F
200    C         standardMessageUnit=errorMessageUnit
201             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
202             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
203    ../tools/genmake2 -of \
204     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
205     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
206    make depend
207    make -j 16
208    cd ~/llc_4320/MITgcm/run_45x45
209    cp ../build/mitgcmuv .
210    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
211    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
212    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
213    mv data.exch2_120x120x10901 data.exch2
214    emacs data
215     tRef =  18.89, 18.89,
216     sRef =  34.84, 34.84,
217     endtime=20.,
218     delR =   1.00,    1.14,
219    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
220    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
221    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
222    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
223    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
224    emacs data.exch2
225    # remove the blank tile list
226    export MPI_BUFS_PER_PROC=1024
227    export MPI_REQUEST_MAX=65536
228    export MPI_GROUP_MAX=1024
229    export MPI_NUM_MEMORY_REGIONS=8
230    export MPI_UNBUFFERED_STDIO=1
231    export MPI_MEMMAP_OFF=1
232    export MPI_UD_TIMEOUT=100
233    mpiexec -n 12000 ./mitgcmuv
234    
235    
236    #############################
237    # generate 48x48 blank tiles
238    qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com
239    module purge
240    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
241    cd ~/llc_4320/MITgcm
242    mkdir run_48x48
243    lfs setstripe -c -1 run_48x48
244    cd build
245    rm *
246    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
247    emacs readtile_mpiio.c
248        tileSizeX = 48;
249        tileSizeY = 48;
250    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
251    emacs SIZE.h
252         &           sNx =  48,
253         &           sNy =  48,
254         &           nSx =  10,
255         &           nPx = 10530,
256         &           Nr  =  2 )
257    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
258    emacs eeboot_minimal.F
259    C         standardMessageUnit=errorMessageUnit
260             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
261             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
262    ../tools/genmake2 -of \
263     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
264     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
265    make depend
266    make -j 16
267    cd ~/llc_4320/MITgcm/run_48x48
268    cp ../build/mitgcmuv .
269    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
270    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
271    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
272    mv data.exch2_120x120x10901 data.exch2
273    emacs data.exch2
274    # remove blankList
275    emacs data
276     tRef =  18.89, 18.89,
277     sRef =  34.84, 34.84,
278     endtime=20.,
279     delR =   1.00,    1.14,
280    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
281    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
282    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
283    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
284    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
285    export MPI_BUFS_PER_PROC=1024
286    export MPI_REQUEST_MAX=65536
287    export MPI_GROUP_MAX=1024
288    export MPI_NUM_MEMORY_REGIONS=8
289    export MPI_UNBUFFERED_STDIO=1
290    export MPI_MEMMAP_OFF=1
291    export MPI_UD_TIMEOUT=100
292    mpiexec -n 12000 ./mitgcmuv
293    
294    
295  ==============  ==============
296    qsub -I -q R3089666 -l select=1750:model=ivy:aoe=sles11,walltime=04:00:00
297    tcsh
298    cd ~/llc_4320/MITgcm
299    mkdir run
300    lfs setstripe -c -1 run
301    cd run
302    cat $PBS_NODEFILE | awk '{for (i=0;i<20;++i) print $0}' > mynodes
303    setenv PBS_NODEFILE mynodes
304    cp /nobackupp8/chenze/run/mitgcmuv_72x72x29297 .
305    ln -sf /nobackupp8/chenze/run/pickup_seaice_0000000360.meta pickup_seaice.0000000180.meta
306    ln -sf /nobackupp8/chenze/run/pickup_seaice_0000000360.data pickup_seaice.0000000180.data
307    ln -sf /nobackupp8/chenze/run/pickup_0000000360.meta pickup.0000000180.meta
308    ln -sf /nobackupp8/chenze/run/pickup_0000000360.data pickup.0000000180.data
309    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
310    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
311    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
312    mv data.exch2_72x72x29297 data.exch2
313    module purge
314    module load  comp-intel/2012.0.032 netcdf/4.0
315    module use -a ~kjtaylor/modulefiles
316    module load sles11sp3/mpt-2.10-nasa201311271217
317    setenv MPI_BUFS_PER_PROC 512
318    setenv MPI_REQUEST_MAX 65536
319    setenv MPI_GROUP_MAX 1024
320    setenv MPI_NUM_MEMORY_REGIONS 8
321    setenv MPI_UNBUFFERED_STDIO 1
322    setenv MPI_MEMMAP_OFF 1
323    export MPI_UD_TIMEOUT=100
324    
325    mpiexec -n 35000 ./mitgcmuv_72x72x29297
326    
327    tail -f STDOUT.00000 | grep advcfl_w
328    
329    
330    ==============
331  cd ~/llc_4320  cd ~/llc_4320
332  cvs co MITgcm_code  cvs co MITgcm_code
333  cvs co MITgcm_contrib/llc_hires/llc_4320  cvs co MITgcm_contrib/llc_hires/llc_4320
334    # set correct tileSizeX and tileSizeY in MITgcm_contrib/llc_hires/llc_4320/cpde-asyn/readtile_mpiio.c
335  cd MITgcm  cd MITgcm
336  module purge  module purge
337  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
# Line 22  cp ../../MITgcm_contrib/llc_hires/llc_43 Line 345  cp ../../MITgcm_contrib/llc_hires/llc_43
345  make depend  make depend
346  make -j 16  make -j 16
347  cd ../run  cd ../run
348  ln -sf ../build/mitgcmuv .  cp ../build/mitgcmuv mitgcmuv_72x72x29297
349  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
350  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
351  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
352  mv data.exch2_72x72x29297 data.exch2  mv data.exch2_72x72x29297 data.exch2
353  export MPI_NUM_MEMORY_REGIONS=256  export MPI_NUM_MEMORY_REGIONS=256
354  mpiexec -n 35000 ./mitgcmuv  mpiexec -n 35000 ./mitgcmuv_72x72x29297
355    
356    tail -f STDOUT.00000 | grep advcfl_w
357    
358  ==============  ==============
359    cd ~/llc_4320
360    cvs co MITgcm_code
361    cvs co MITgcm_contrib/llc_hires/llc_4320
362    # set correct tileSizeX and tileSizeY in MITgcm_contrib/llc_hires/llc_4320/cpde-asyn/readtile_mpiio.c
363    cd MITgcm
364    module purge
365    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
366    mkdir build run_180x180x5015
367    lfs setstripe -c -1 run_180x180x5015
368    cd build
369    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_180x180x5015 SIZE.h
370    ../tools/genmake2 -of \
371     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
372     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
373    make depend
374    make -j 16
375    cd ../run_180x180x5015
376    cp ../build/mitgcmuv mitgcmuv_180x180x5015
377    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
378    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
379    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
380    mv data.exch2_72x72x29297 data.exch2
381    export MPI_NUM_MEMORY_REGIONS=256
382    mpiexec -n 6000 ./mitgcmuv_180x180x5015
383    
384    ==============
385  look at output  look at output
386    
387  for ts=[0 120 600:10:980 1080:120:2280]  for ts=[0 120 600:10:980 1080:120:2280]
# Line 41  for ts=[0 120 600:10:980 1080:120:2280] Line 391  for ts=[0 120 600:10:980 1080:120:2280]
391      pause(.1)      pause(.1)
392  end  end
393    
 ==============  
394    
395    ==============
396  to determine empty tiles:  to determine empty tiles:
397  grep Empty STDOUT.*  grep Empty STDOUT.*
398    
399  =============  =============
   
400  memory requirements:  memory requirements:
401  nPx  sNx sNy nSx cpu node0        total  nPx  sNx sNy nSx cpu node0        total
402  3744 180 180   2 san 22,106,128kb 5,195,641,224kb - node ran out of memory and crashed  3744 180 180   2 san 22,106,128kb 5,195,641,224kb - node ran out of memory and crashed
# Line 55  nPx  sNx sNy nSx cpu node0        total Line 404  nPx  sNx sNy nSx cpu node0        total
404  7488 180 180   1 san  7488 180 180   1 san
405    
406  =============  =============
   
407  2               =    2  2               =    2
408  3               =    3  3               =    3
409  2*2             =    4  2*2             =    4
Legend:
Removed from v.1.8  
changed lines
  Added in v.1.16
  ViewVC Help
Powered by ViewVC 1.1.22