/[MITgcm]/MITgcm_contrib/llc_hires/llc_1080/readme.txt
ViewVC logotype

Annotation of /MITgcm_contrib/llc_hires/llc_1080/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph

Revision 1.12 - (hide annotations) (download)
Wed Jun 7 00:32:51 2017 UTC (8 years, 1 month ago) by dimitri
Branch: MAIN
Changes since 1.11: +27 -0 lines
File MIME type: text/plain 
Updated to checkpoint66h. Similar but not identical to checkpoint64p.

For checkpoint66h:
bash-3.2$ grep advcfl_W STDOUT.0000 | head
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   8.0765112988021E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.6646790127962E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.2944741238975E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.0442152223351E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   5.3384062112729E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.9344950973214E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.0813966489876E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.2877764643888E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.8223244494440E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.6377593176494E-01

For checkpoint64p:
bash-3.2$ grep advcfl_W STDOUT.0000 | head
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   8.0765112988021E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.6646790128502E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.2944740988061E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.0440425691133E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   5.3384765401778E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.9344949519849E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.0813964719293E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.2877763699241E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.8221353737459E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.6377680028628E-01
1 dimitri 1.11 ==============
2 dimitri 1.12 # interactive 90x90x1342 tile configuration with newer MITgcm code
3     cd ~/llc_1080
4     cvs co MITgcm_contrib/llc_hires/llc_1080
5     cvs co -r checkpoint66h MITgcm_code
6     qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
7     module purge
8     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
9     cd ~/llc_1080/MITgcm
10     mkdir build run
11     lfs setstripe -c -1 run
12     cd build
13     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
14     ../tools/genmake2 -of \
15     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
16     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
17     make depend
18     make -j 56
19     cd ../run
20     ln -sf ../build/mitgcmuv .
21     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
22     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
23     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
24     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
25     mv data.exch2_90x90x1342 data.exch2
26     mpiexec -n 1342 ./mitgcmuv &
27    
28     ==============
29 dimitri 1.11 # interactive 90x90x1342 tile configuration from scratch
30     cd ~/llc_1080
31     cvs co MITgcm_contrib/llc_hires/llc_1080
32     cvs co -r checkpoint64p MITgcm_code
33     qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
34     module purge
35     module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
36     cd ~/llc_1080/MITgcm
37     mkdir build run
38     lfs setstripe -c -1 run
39     cd build
40     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
41     ../tools/genmake2 -of \
42     ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
43     -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
44     make depend
45     make -j 16
46     cd ../run
47     ln -sf ../build/mitgcmuv .
48     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
49     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
50     ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
51     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
52     mv data.exch2_90x90x1342 data.exch2
53     mpiexec -n 1342 ./mitgcmuv &
54    
55     =====================
56 dimitri 1.1 For interactive session, Ivy Bridge nodes:
57 dimitri 1.7 qsub -I -q devel -l select=170:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
58     qsub -I -q long -l select=170:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
59     qsub -I -q long -l select=170:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M email
60 dimitri 1.1
61 dimitri 1.6 ==============
62 dimitri 1.7 # 60x60x2872 tile configuration
63 dimitri 1.6
64     cd ~/llc_1080
65 dimitri 1.10 cvs co -r checkpoint64p MITgcm_code
66 dimitri 1.6 cvs co MITgcm_contrib/llc_hires/llc_1080
67     cd MITgcm
68     module purge
69 dimitri 1.7 module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
70 dimitri 1.6 mkdir build run
71     lfs setstripe -c -1 run
72     cd build
73     cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_60x60x2872 SIZE.h
74     ../tools/genmake2 -of \
75     ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
76     '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
77     make depend
78     make -j 16
79 dimitri 1.9 cd ~/llc_1080/MITgcm/run
80 dimitri 1.6 ln -sf ../build/mitgcmuv .
81     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
82     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
83     cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
84     mv data.exch2_60x60x2872 data.exch2
85 dimitri 1.8 export MPI_NUM_MEMORY_REGIONS=256
86 dimitri 1.6 mpiexec -n 3400 ./mitgcmuv
87 dimitri 1.1
88     ==============
89 dimitri 1.7 # 90x90x1342 tile configuration
90 dimitri 1.1
91 dimitri 1.3 cd ~/llc_1080
92 dimitri 1.1 cvs co MITgcm_code
93 dimitri 1.3 cvs co MITgcm_contrib/llc_hires/llc_1080
94 dimitri 1.1 cd MITgcm
95     module purge
96 dimitri 1.6 module load comp-intel/2011.2 mpi-sgi/mpt.2.06r6 netcdf/4.0
97 dimitri 1.1 mkdir build run
98 dimitri 1.3 lfs setstripe -c -1 run
99 dimitri 1.1 cd build
100 dimitri 1.3 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
101     ../tools/genmake2 -of \
102     ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
103     '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
104 dimitri 1.1 make depend
105 dimitri 1.6 make -j 16
106 dimitri 1.1 cd ../run
107     ln -sf ../build/mitgcmuv .
108     ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
109 dimitri 1.5 #ln -sf /nobackup/dmenemen/forcing/era_interim/EIG_*_2* .
110     #ln -sf /nobackup/dmenemen/forcing/era_interim_corrected/EIG_dlw_sub5p_2* .
111     ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
112 dimitri 1.3 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
113     mv data.exch2_90x90x1342 data.exch2
114 dimitri 1.8 export MPI_NUM_MEMORY_REGIONS=256
115 dimitri 1.3 mpiexec -n 1600 ./mitgcmuv
116 dimitri 1.1
117     ==============
118    
119     look at output
120    
121 dimitri 1.4 for ts=[50760]
122 dimitri 1.1 fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],1080);
123     clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar
124     title(ts)
125     pause(.1)
126     end
127    
128 dimitri 1.4 ts=50760;
129     for fld={'S''T','U','V','W','HEFF'}
130 dimitri 1.1 tmp=quikread_llc([fld{1} '.' myint2str(ts,10) '.data'],1080);
131     clf,quikplot_llc(tmp),thincolorbar
132     title(fld{1})
133     pause
134     end
135    
136     ==============
137    
138     to determine empty tiles:
139     grep Empty STDOUT.*
140    
141     ==============
142    
143     memory requirements:
144     nPx sNx sNy nSx node0 total rank0 rank1
145     702 60 60 6 17,642,924kb 770,168,568kb
146     468 60 60 9 22,083,004kb 643,301,784kb
147     468 180 180 1 24,719,776kb 720,415,552kb 1678MB 1563MB
148     351 60 60 12 26,660,020kb 582,815,544kb
149     351 120 120 3 25,110,324kb 548,819,772kb 1711MB 1586MB
150     351 120 120 3 6,543,780kb 56,729,484kb - with globalFiles - run out of time
151     325 54 54 16 29,029,744kb 587,831,344kb
152     325 216 216 1 29,249,232kb 426,554,484kb - node ran out of memory and crashed
153     234 90 90 8 30,590,952kb 431,252,200kb - node ran out of memory and crashed
154     234 180 180 2 27,682,360kb 307,181,152kb - node ran out of memory and crashed
155    
156     =============
157     factor(3240) = 2 2 2 3 3 3 3 5
158     factor(1080) = 2 2 2 3 3 3 5
159    
160     2 = 2
161     3 = 3
162     2*2 = 4
163     5 = 5
164     2*3 = 6
165     2*2*2 = 8
166     3*3 = 9
167     2*5 = 10
168     2*2*3 = 12
169     3*5 = 15
170     2*3*3 = 18
171     2*2*5 = 20
172     2*2*2*3 = 24
173     3*3*3 = 27
174     2*3*5 = 30
175     2*2*3*3 = 36 * 30
176     2*2*2*5 = 40 * 27
177     3*3*5 = 45 * 24
178     2*3*3*3 = 54 * 20
179     2*2*3*5 = 60 * 18
180     2*2*2*3*3 = 72 * 15
181     2*3*3*5 = 90 * 12
182     2*2*3*3*3 = 108 * 10
183     2*2*2*3*5 = 120 * 9
184     3*3*3*5 = 135 * 8
185     2*2*3*3*5 = 180 * 6
186     2*2*2*3*3*3 = 216 * 5
187     2*3*3*3*5 = 270 * 4
188     2*2*2*3*3*5 = 360 * 3
189     2*2*3*3*3*5 = 540 * 2
  ViewVC Help
Powered by ViewVC 1.1.22