/[MITgcm]/MITgcm_contrib/llc_hires/llc_4320/readme.txt
ViewVC logotype

Contents of /MITgcm_contrib/llc_hires/llc_4320/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.12 - (show annotations) (download)
Mon Feb 10 15:34:38 2014 UTC (11 years, 5 months ago) by dimitri
Branch: MAIN
Changes since 1.11: +107 -14 lines
File MIME type: text/plain
preparing 48x48-tile 70K configuration

1 For interactive session, Ivy Bridge nodes:
2 qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
3 qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email
4 qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
5
6 #############################
7 # 90x90x19023 configuration
8
9 qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
10 module purge
11 module load comp-intel/2012.0.032 netcdf/4.0
12 module use -a ~kjtaylor/modulefiles
13 module load sles11sp3/mpt-2.10-nasa201311271217
14 cd ~/llc_4320
15 cvs co MITgcm_code
16 cvs co MITgcm_contrib/llc_hires/llc_4320
17 cd MITgcm
18 mkdir build run
19 lfs setstripe -c -1 run
20 cd build
21 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
22 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
23 emacs readtile_mpiio.c
24 tileSizeX = 90;
25 tileSizeY = 90;
26 ../tools/genmake2 -of \
27 ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
28 '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
29 make depend
30 make -j 16
31
32 cd ~/llc_4320/MITgcm/run
33 cp ../build/mitgcmuv mitgcmuv_90x90x19023
34 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
35 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
36 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
37 mv data.exch2_90x90x19023 data.exch2
38 emacs data
39
40 export MPI_BUFS_PER_PROC=1024
41 export MPI_REQUEST_MAX=65536
42 export MPI_GROUP_MAX=1024
43 export MPI_NUM_MEMORY_REGIONS=8
44 export MPI_UNBUFFERED_STDIO=1
45 export MPI_MEMMAP_OFF=1
46 export MPI_UD_TIMEOUT=100
47 mpiexec -n 20400 ./mitgcmuv_90x90x19023
48
49 tail -f STDOUT.00000 | grep advcfl_W
50
51 #############################
52 # generate 60x60 blank tiles
53 qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
54 module purge
55 module load comp-intel/2012.0.032 netcdf/4.0
56 module use -a ~kjtaylor/modulefiles
57 module load sles11sp3/mpt-2.10-nasa201311271217
58 cd ~/llc_4320/MITgcm
59 mkdir run_60x60
60 lfs setstripe -c -1 run_60x60
61 cd build
62 rm *
63 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
64 emacs readtile_mpiio.c
65 tileSizeX = 60;
66 tileSizeY = 60;
67 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
68 emacs SIZE.h
69 & sNx = 60,
70 & sNy = 60,
71 & nSx = 6,
72 & nPx = 11232,
73 & Nr = 2 )
74 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
75 emacs eeboot_minimal.F
76 C standardMessageUnit=errorMessageUnit
77 WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
78 OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
79 ../tools/genmake2 -of \
80 ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
81 '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
82 make depend
83 make -j 16
84 cd ~/llc_4320/MITgcm/run_60x60
85 cp ../build/mitgcmuv .
86 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
87 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
88 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
89 mv data.exch2_120x120x10901 data.exch2
90 emacs data
91 tRef = 18.89, 18.89,
92 sRef = 34.84, 34.84,
93 endtime=20.,
94 delR = 1.00, 1.14,
95 # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
96 # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
97 # uVelInitFile ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
98 # vVelInitFile ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
99 # pSurfInitFile ='ETAN_llc2160_10sep2011_4320x56160_r4',
100 emacs data.exch2
101 # remove the blank tile list
102 export MPI_BUFS_PER_PROC=1024
103 export MPI_REQUEST_MAX=65536
104 export MPI_GROUP_MAX=1024
105 export MPI_NUM_MEMORY_REGIONS=8
106 export MPI_UNBUFFERED_STDIO=1
107 export MPI_MEMMAP_OFF=1
108 export MPI_UD_TIMEOUT=100
109 mpiexec -n 12000 ./mitgcmuv
110
111 #############################
112 # generate 45x45 blank tiles
113 qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
114 module purge
115 module load comp-intel/2012.0.032 netcdf/4.0
116 module use -a ~kjtaylor/modulefiles
117 module load sles11sp3/mpt-2.10-nasa201311271217
118 cd ~/llc_4320/MITgcm
119 mkdir run_45x45
120 lfs setstripe -c -1 run_45x45
121 cd build
122 rm *
123 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
124 emacs readtile_mpiio.c
125 tileSizeX = 45;
126 tileSizeY = 45;
127 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
128 emacs SIZE.h
129 & sNx = 45,
130 & sNy = 45,
131 & nSx = 12,
132 & nPx = 9984,
133 & Nr = 2 )
134 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
135 emacs eeboot_minimal.F
136 C standardMessageUnit=errorMessageUnit
137 WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
138 OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
139 ../tools/genmake2 -of \
140 ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
141 '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
142 make depend
143 make -j 16
144 cd ~/llc_4320/MITgcm/run_45x45
145 cp ../build/mitgcmuv .
146 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
147 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
148 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
149 mv data.exch2_120x120x10901 data.exch2
150 emacs data
151 tRef = 18.89, 18.89,
152 sRef = 34.84, 34.84,
153 endtime=20.,
154 delR = 1.00, 1.14,
155 # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
156 # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
157 # uVelInitFile ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
158 # vVelInitFile ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
159 # pSurfInitFile ='ETAN_llc2160_10sep2011_4320x56160_r4',
160 emacs data.exch2
161 # remove the blank tile list
162 export MPI_BUFS_PER_PROC=1024
163 export MPI_REQUEST_MAX=65536
164 export MPI_GROUP_MAX=1024
165 export MPI_NUM_MEMORY_REGIONS=8
166 export MPI_UNBUFFERED_STDIO=1
167 export MPI_MEMMAP_OFF=1
168 export MPI_UD_TIMEOUT=100
169 mpiexec -n 12000 ./mitgcmuv
170
171 #############################
172 # generate 48x48 blank tiles
173 qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com
174 module purge
175 module load comp-intel/2012.0.032 netcdf/4.0
176 module use -a ~kjtaylor/modulefiles
177 module load sles11sp3/mpt-2.10-nasa201311271217
178 cd ~/llc_4320/MITgcm
179 mkdir run_48x48
180 lfs setstripe -c -1 run_48x48
181 cd build
182 rm *
183 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
184 emacs readtile_mpiio.c
185 tileSizeX = 48;
186 tileSizeY = 48;
187 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
188 emacs SIZE.h
189 & sNx = 48,
190 & sNy = 48,
191 & nSx = 10,
192 & nPx = 10530,
193 & Nr = 2 )
194 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
195 emacs eeboot_minimal.F
196 C standardMessageUnit=errorMessageUnit
197 WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
198 OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
199 ../tools/genmake2 -of \
200 ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
201 '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
202 make depend
203 make -j 16
204 cd ~/llc_4320/MITgcm/run_48x48
205 cp ../build/mitgcmuv .
206 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
207 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
208 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
209 mv data.exch2_120x120x10901 data.exch2
210 emacs data.exch2
211 # remove blankList
212 emacs data
213 tRef = 18.89, 18.89,
214 sRef = 34.84, 34.84,
215 endtime=20.,
216 delR = 1.00, 1.14,
217 # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
218 # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
219 # uVelInitFile ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
220 # vVelInitFile ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
221 # pSurfInitFile ='ETAN_llc2160_10sep2011_4320x56160_r4',
222 export MPI_BUFS_PER_PROC=1024
223 export MPI_REQUEST_MAX=65536
224 export MPI_GROUP_MAX=1024
225 export MPI_NUM_MEMORY_REGIONS=8
226 export MPI_UNBUFFERED_STDIO=1
227 export MPI_MEMMAP_OFF=1
228 export MPI_UD_TIMEOUT=100
229 mpiexec -n 12000 ./mitgcmuv
230
231 #############################
232 # 120x120x10901 configuration
233
234 qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
235 module purge
236 module load comp-intel/2012.0.032 netcdf/4.0
237 module use -a ~kjtaylor/modulefiles
238 module load sles11sp3/mpt-2.10-nasa201311271217
239 cd ~/llc_4320
240 cvs co MITgcm_code
241 cvs co MITgcm_contrib/llc_hires/llc_4320
242 cd MITgcm
243 mkdir build run
244 lfs setstripe -c -1 run
245 cd build
246 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
247 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
248 emacs readtile_mpiio.c
249 tileSizeX = 120;
250 tileSizeY = 120;
251 ../tools/genmake2 -of \
252 ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
253 '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
254 make depend
255 make -j 16
256
257 cd ~/llc_4320/MITgcm/run
258 cp ../build/mitgcmuv mitgcmuv_120x120x10901
259 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
260 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
261 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
262 mv data.exch2_120x120x10901 data.exch2
263 emacs data
264
265 export MPI_BUFS_PER_PROC=1024
266 export MPI_REQUEST_MAX=65536
267 export MPI_GROUP_MAX=1024
268 export MPI_NUM_MEMORY_REGIONS=8
269 export MPI_UNBUFFERED_STDIO=1
270 export MPI_MEMMAP_OFF=1
271 export MPI_UD_TIMEOUT=100
272 mpiexec -n 12000 ./mitgcmuv_120x120x10901
273
274 tail -f STDOUT.00000 | grep advcfl_W
275
276 ==============
277
278 qsub -I -q R3089666 -l select=1750:model=ivy:aoe=sles11,walltime=04:00:00
279 tcsh
280 cd ~/llc_4320/MITgcm
281 mkdir run
282 lfs setstripe -c -1 run
283 cd run
284 cat $PBS_NODEFILE | awk '{for (i=0;i<20;++i) print $0}' > mynodes
285 setenv PBS_NODEFILE mynodes
286 cp /nobackupp8/chenze/run/mitgcmuv_72x72x29297 .
287 ln -sf /nobackupp8/chenze/run/pickup_seaice_0000000360.meta pickup_seaice.0000000180.meta
288 ln -sf /nobackupp8/chenze/run/pickup_seaice_0000000360.data pickup_seaice.0000000180.data
289 ln -sf /nobackupp8/chenze/run/pickup_0000000360.meta pickup.0000000180.meta
290 ln -sf /nobackupp8/chenze/run/pickup_0000000360.data pickup.0000000180.data
291 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
292 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
293 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
294 mv data.exch2_72x72x29297 data.exch2
295 module purge
296 module load comp-intel/2012.0.032 netcdf/4.0
297 module use -a ~kjtaylor/modulefiles
298 module load sles11sp3/mpt-2.10-nasa201311271217
299 setenv MPI_BUFS_PER_PROC 512
300 setenv MPI_REQUEST_MAX 65536
301 setenv MPI_GROUP_MAX 1024
302 setenv MPI_NUM_MEMORY_REGIONS 8
303 setenv MPI_UNBUFFERED_STDIO 1
304 setenv MPI_MEMMAP_OFF 1
305 export MPI_UD_TIMEOUT=100
306
307 mpiexec -n 35000 ./mitgcmuv_72x72x29297
308
309 tail -f STDOUT.00000 | grep advcfl_w
310
311 ==============
312
313 cd ~/llc_4320
314 cvs co MITgcm_code
315 cvs co MITgcm_contrib/llc_hires/llc_4320
316 # set correct tileSizeX and tileSizeY in MITgcm_contrib/llc_hires/llc_4320/cpde-asyn/readtile_mpiio.c
317 cd MITgcm
318 module purge
319 module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
320 mkdir build run
321 lfs setstripe -c -1 run
322 cd build
323 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_72x72x29297 SIZE.h
324 ../tools/genmake2 -of \
325 ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
326 '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
327 make depend
328 make -j 16
329 cd ../run
330 cp ../build/mitgcmuv mitgcmuv_72x72x29297
331 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
332 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
333 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
334 mv data.exch2_72x72x29297 data.exch2
335 export MPI_NUM_MEMORY_REGIONS=256
336 mpiexec -n 35000 ./mitgcmuv_72x72x29297
337
338 tail -f STDOUT.00000 | grep advcfl_w
339
340 ==============
341
342 cd ~/llc_4320
343 cvs co MITgcm_code
344 cvs co MITgcm_contrib/llc_hires/llc_4320
345 # set correct tileSizeX and tileSizeY in MITgcm_contrib/llc_hires/llc_4320/cpde-asyn/readtile_mpiio.c
346 cd MITgcm
347 module purge
348 module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
349 mkdir build run_180x180x5015
350 lfs setstripe -c -1 run_180x180x5015
351 cd build
352 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_180x180x5015 SIZE.h
353 ../tools/genmake2 -of \
354 ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
355 '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
356 make depend
357 make -j 16
358 cd ../run_180x180x5015
359 cp ../build/mitgcmuv mitgcmuv_180x180x5015
360 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
361 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
362 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
363 mv data.exch2_72x72x29297 data.exch2
364 export MPI_NUM_MEMORY_REGIONS=256
365 mpiexec -n 6000 ./mitgcmuv_180x180x5015
366
367 ==============
368
369 look at output
370
371 for ts=[0 120 600:10:980 1080:120:2280]
372 fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],4320);
373 clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar
374 title(ts)
375 pause(.1)
376 end
377
378 ==============
379
380 to determine empty tiles:
381 grep Empty STDOUT.*
382
383 =============
384
385 memory requirements:
386 nPx sNx sNy nSx cpu node0 total
387 3744 180 180 2 san 22,106,128kb 5,195,641,224kb - node ran out of memory and crashed
388 5616 120 120 3 san - node ran out of memory and crashed
389 7488 180 180 1 san
390
391 =============
392
393 2 = 2
394 3 = 3
395 2*2 = 4
396 5 = 5
397 2*3 = 6
398 2*2*2 = 8
399 3*3 = 9
400 2*5 = 10
401 2*2*3 = 12
402 3*5 = 15
403 2*2*2*2 = 16
404 2*3*3 = 18
405 2*2*5 = 20
406 2*2*2*3 = 24
407 3*3*3 = 27
408 2*3*5 = 30
409 2*2*2*2*2 = 32
410 2*2*3*3 = 36
411 2*2*2*5 = 40
412 3*3*5 = 45
413 2*2*2*2*3 = 48
414 2*3*3*3 = 54
415 2*2*3*5 = 60
416 2*2*2*3*3 = 72 * 60
417 2*2*2*2*5 = 80 * 54
418 2*3*3*5 = 90 * 48
419 2*2*2*2*2*3 = 96 * 45
420 2*2*3*3*3 = 108 * 40
421 2*2*2*3*5 = 120 * 36
422 3*3*3*5 = 135 * 32
423 2*2*2*2*3*3 = 144 * 30
424 2*2*2*2*2*5 = 160 * 27
425 2*2*3*3*5 = 180 * 24
426 2*2*2*3*3*3 = 216 * 20
427 2*2*2*2*3*5 = 240 * 18
428 2*3*3*3*5 = 270 * 16
429 2*2*2*2*2*3*3 = 288 * 15
430 2*2*2*3*3*5 = 360 * 12
431 2*2*2*2*3*3*3 = 432 * 10
432 2*2*2*2*2*3*5 = 480 * 9
433 2*2*3*3*3*5 = 540 * 8
434 2*2*2*2*3*3*5 = 720 * 6
435 2*2*2*2*2*3*3*3 = 864 * 5
436 2*2*2*3*3*3*5 = 1080 * 4
437 2*2*2*2*2*3*3*5 = 1440 * 3
438 2*2*2*2*3*3*3*5 = 2160 * 2

  ViewVC Help
Powered by ViewVC 1.1.22