/[MITgcm]/MITgcm_contrib/llc_hires/llc_1080/readme.txt
ViewVC logotype

Annotation of /MITgcm_contrib/llc_hires/llc_1080/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.14 - (hide annotations) (download)
Thu Feb 21 00:48:32 2019 UTC (6 years, 5 months ago) by dimitri
Branch: MAIN
Changes since 1.13: +136 -1 lines
File MIME type: text/plain
adding instructions for a few more tile sizes

1 dimitri 1.11 ==============
2 dimitri 1.14 # interactive 30x30x16848 tile configuration with newer MITgcm code
3     cd ~/llc_1080
4     cvs co MITgcm_contrib/llc_hires/llc_1080
5     cvs co -r checkpoint66h MITgcm_code
6     qsub -I -q devel -l select=602:ncpus=28:model=bro,walltime=2:00:00 -m abe
7     module purge
8     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
9     cd ~/llc_1080/MITgcm
10     mkdir build run
11     lfs setstripe -c -1 run
12     cd build
13     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_30x30 SIZE.h
14     ../tools/genmake2 -of \
15     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
16     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
17     make depend
18     make -j 56
19     cd ../run
20     ln -sf ../build/mitgcmuv .
21     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
22     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
23     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
24     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
25     mv data.exch2_30x30x16848 data.exch2
26     mpiexec -n 16848 ./mitgcmuv &
27    
28     ==============
29     # interactive 36x36x11700 tile configuration with newer MITgcm code
30     cd ~/llc_1080
31     cvs co MITgcm_contrib/llc_hires/llc_1080
32     cvs co -r checkpoint66h MITgcm_code
33     qsub -I -q devel -l select=418:ncpus=28:model=bro,walltime=2:00:00 -m abe
34     module purge
35     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
36     cd ~/llc_1080/MITgcm
37     mkdir build run
38     lfs setstripe -c -1 run
39     cd build
40     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_36x36 SIZE.h
41     ../tools/genmake2 -of \
42     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
43     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
44     make depend
45     make -j 56
46     cd ../run
47     ln -sf ../build/mitgcmuv .
48     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
49     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
50     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
51     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
52     mv data.exch2_36x36x11700 data.exch2
53     mpiexec -n 11700 ./mitgcmuv &
54    
55     ==============
56     # interactive 40x40x9477 tile configuration with newer MITgcm code
57     cd ~/llc_1080
58     cvs co MITgcm_contrib/llc_hires/llc_1080
59     cvs co -r checkpoint66h MITgcm_code
60     qsub -I -q devel -l select=339:ncpus=28:model=bro,walltime=2:00:00 -m abe
61     module purge
62     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
63     cd ~/llc_1080/MITgcm
64     mkdir build run
65     lfs setstripe -c -1 run
66     cd build
67     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_40x40 SIZE.h
68     ../tools/genmake2 -of \
69     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
70     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
71     make depend
72     make -j 56
73     cd ../run
74     ln -sf ../build/mitgcmuv .
75     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
76     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
77     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
78     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
79     mv data.exch2_40x40x9477 data.exch2
80     mpiexec -n 9477 ./mitgcmuv &
81    
82     ==============
83     # interactive 45x45x7488 tile configuration with newer MITgcm code
84     cd ~/llc_1080
85     cvs co MITgcm_contrib/llc_hires/llc_1080
86     cvs co -r checkpoint66h MITgcm_code
87     qsub -I -q devel -l select=268:ncpus=28:model=bro,walltime=2:00:00 -m abe
88     module purge
89     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
90     cd ~/llc_1080/MITgcm
91     mkdir build run
92     lfs setstripe -c -1 run
93     cd build
94     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_45x45 SIZE.h
95     ../tools/genmake2 -of \
96     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
97     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
98     make depend
99     make -j 56
100     cd ../run
101     ln -sf ../build/mitgcmuv .
102     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
103     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
104     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
105     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
106     mv data.exch2_45x45x7488 data.exch2
107     mpiexec -n 7488 ./mitgcmuv &
108    
109     ==============
110     # interactive 54x54x5200 tile configuration with newer MITgcm code
111     cd ~/llc_1080
112     cvs co MITgcm_contrib/llc_hires/llc_1080
113     cvs co -r checkpoint66h MITgcm_code
114     qsub -I -q devel -l select=268:ncpus=28:model=bro,walltime=2:00:00 -m abe
115     module purge
116     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
117     cd ~/llc_1080/MITgcm
118     mkdir build run
119     lfs setstripe -c -1 run
120     cd build
121     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_54x54 SIZE.h
122     ../tools/genmake2 -of \
123     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
124     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
125     make depend
126     make -j 56
127     cd ../run
128     ln -sf ../build/mitgcmuv .
129     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
130     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
131     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
132     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
133     mv data.exch2_54x54x5200 data.exch2
134     mpiexec -n 5200 ./mitgcmuv &
135    
136     ==============
137     # Interactive 90x90x1342 tile configuration with newer MITgcm code
138 dimitri 1.12 cd ~/llc_1080
139     cvs co MITgcm_contrib/llc_hires/llc_1080
140     cvs co -r checkpoint66h MITgcm_code
141     qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
142     module purge
143     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
144     cd ~/llc_1080/MITgcm
145     mkdir build run
146     lfs setstripe -c -1 run
147     cd build
148     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
149     ../tools/genmake2 -of \
150     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
151     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
152     make depend
153     make -j 56
154     cd ../run
155     ln -sf ../build/mitgcmuv .
156     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
157     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
158     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
159     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
160     mv data.exch2_90x90x1342 data.exch2
161     mpiexec -n 1342 ./mitgcmuv &
162    
163     ==============
164 dimitri 1.11 # interactive 90x90x1342 tile configuration from scratch
165     cd ~/llc_1080
166     cvs co MITgcm_contrib/llc_hires/llc_1080
167     cvs co -r checkpoint64p MITgcm_code
168     qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
169     module purge
170     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
171     cd ~/llc_1080/MITgcm
172     mkdir build run
173     lfs setstripe -c -1 run
174     cd build
175     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
176     ../tools/genmake2 -of \
177     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
178     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
179     make depend
180     make -j 16
181     cd ../run
182     ln -sf ../build/mitgcmuv .
183     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
184     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
185     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
186     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
187     mv data.exch2_90x90x1342 data.exch2
188     mpiexec -n 1342 ./mitgcmuv &
189    
190     =====================
191 dimitri 1.1 For interactive session, Ivy Bridge nodes:
192 dimitri 1.13 qsub -I -q debug -l select=48:ncpus=28:model=bro,walltime=02:00:00 -m abe -M email
193 dimitri 1.7 qsub -I -q devel -l select=170:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
194     qsub -I -q long -l select=170:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
195     qsub -I -q long -l select=170:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M email
196 dimitri 1.1
197 dimitri 1.6 ==============
198 dimitri 1.7 # 60x60x2872 tile configuration
199 dimitri 1.6
200     cd ~/llc_1080
201 dimitri 1.10 cvs co -r checkpoint64p MITgcm_code
202 dimitri 1.6 cvs co MITgcm_contrib/llc_hires/llc_1080
203     cd MITgcm
204     module purge
205 dimitri 1.7 module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
206 dimitri 1.6 mkdir build run
207     lfs setstripe -c -1 run
208     cd build
209     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_60x60x2872 SIZE.h
210     ../tools/genmake2 -of \
211     ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
212     '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
213     make depend
214     make -j 16
215 dimitri 1.9 cd ~/llc_1080/MITgcm/run
216 dimitri 1.6 ln -sf ../build/mitgcmuv .
217     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
218     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
219     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
220     mv data.exch2_60x60x2872 data.exch2
221 dimitri 1.8 export MPI_NUM_MEMORY_REGIONS=256
222 dimitri 1.6 mpiexec -n 3400 ./mitgcmuv
223 dimitri 1.1
224     ==============
225 dimitri 1.7 # 90x90x1342 tile configuration
226 dimitri 1.1
227 dimitri 1.3 cd ~/llc_1080
228 dimitri 1.1 cvs co MITgcm_code
229 dimitri 1.3 cvs co MITgcm_contrib/llc_hires/llc_1080
230 dimitri 1.1 cd MITgcm
231     module purge
232 dimitri 1.6 module load comp-intel/2011.2 mpi-sgi/mpt.2.06r6 netcdf/4.0
233 dimitri 1.1 mkdir build run
234 dimitri 1.3 lfs setstripe -c -1 run
235 dimitri 1.1 cd build
236 dimitri 1.3 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
237     ../tools/genmake2 -of \
238     ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
239     '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
240 dimitri 1.1 make depend
241 dimitri 1.6 make -j 16
242 dimitri 1.1 cd ../run
243     ln -sf ../build/mitgcmuv .
244     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
245 dimitri 1.5 #ln -sf /nobackup/dmenemen/forcing/era_interim/EIG_*_2* .
246     #ln -sf /nobackup/dmenemen/forcing/era_interim_corrected/EIG_dlw_sub5p_2* .
247     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
248 dimitri 1.3 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
249     mv data.exch2_90x90x1342 data.exch2
250 dimitri 1.8 export MPI_NUM_MEMORY_REGIONS=256
251 dimitri 1.3 mpiexec -n 1600 ./mitgcmuv
252 dimitri 1.1
253     ==============
254 dimitri 1.13 # 90x90x1342 tile test configuration
255    
256     cd ~/llc_1080_test
257     cvs co MITgcm_code
258     cvs co MITgcm_contrib/llc_hires/llc_1080
259     cd MITgcm
260     module purge
261     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
262     mkdir build run
263     lfs setstripe -c -1 run
264     cd build
265     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
266     ../tools/genmake2 -of \
267     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
268     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
269     make depend
270     make -j 16
271     cd ../run
272     ln -sf ../build/mitgcmuv .
273     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
274     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
275     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
276     mv data.exch2_90x90x1342 data.exch2
277     mpiexec -n 1342 ./mitgcmuv
278    
279    
280     ==============
281 dimitri 1.1
282     look at output
283    
284 dimitri 1.4 for ts=[50760]
285 dimitri 1.1 fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],1080);
286     clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar
287     title(ts)
288     pause(.1)
289     end
290    
291 dimitri 1.4 ts=50760;
292     for fld={'S''T','U','V','W','HEFF'}
293 dimitri 1.1 tmp=quikread_llc([fld{1} '.' myint2str(ts,10) '.data'],1080);
294     clf,quikplot_llc(tmp),thincolorbar
295     title(fld{1})
296     pause
297     end
298    
299     ==============
300    
301     to determine empty tiles:
302     grep Empty STDOUT.*
303    
304     ==============
305    
306     memory requirements:
307     nPx sNx sNy nSx node0 total rank0 rank1
308     702 60 60 6 17,642,924kb 770,168,568kb
309     468 60 60 9 22,083,004kb 643,301,784kb
310     468 180 180 1 24,719,776kb 720,415,552kb 1678MB 1563MB
311     351 60 60 12 26,660,020kb 582,815,544kb
312     351 120 120 3 25,110,324kb 548,819,772kb 1711MB 1586MB
313     351 120 120 3 6,543,780kb 56,729,484kb - with globalFiles - run out of time
314     325 54 54 16 29,029,744kb 587,831,344kb
315     325 216 216 1 29,249,232kb 426,554,484kb - node ran out of memory and crashed
316     234 90 90 8 30,590,952kb 431,252,200kb - node ran out of memory and crashed
317     234 180 180 2 27,682,360kb 307,181,152kb - node ran out of memory and crashed
318    
319     =============
320     factor(3240) = 2 2 2 3 3 3 3 5
321     factor(1080) = 2 2 2 3 3 3 5
322    
323     2 = 2
324     3 = 3
325     2*2 = 4
326     5 = 5
327     2*3 = 6
328     2*2*2 = 8
329     3*3 = 9
330     2*5 = 10
331     2*2*3 = 12
332     3*5 = 15
333     2*3*3 = 18
334     2*2*5 = 20
335     2*2*2*3 = 24
336     3*3*3 = 27
337     2*3*5 = 30
338     2*2*3*3 = 36 * 30
339     2*2*2*5 = 40 * 27
340     3*3*5 = 45 * 24
341     2*3*3*3 = 54 * 20
342     2*2*3*5 = 60 * 18
343     2*2*2*3*3 = 72 * 15
344     2*3*3*5 = 90 * 12
345     2*2*3*3*3 = 108 * 10
346     2*2*2*3*5 = 120 * 9
347     3*3*3*5 = 135 * 8
348     2*2*3*3*5 = 180 * 6
349     2*2*2*3*3*3 = 216 * 5
350     2*3*3*3*5 = 270 * 4
351     2*2*2*3*3*5 = 360 * 3
352     2*2*3*3*3*5 = 540 * 2

  ViewVC Help
Powered by ViewVC 1.1.22