/[MITgcm]/MITgcm_contrib/llc_hires/llc_4320/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_4320/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.9 by dimitri, Fri Jan 10 20:52:36 2014 UTC revision 1.13 by dimitri, Fri Jun 27 13:56:32 2014 UTC
# Line 2  For interactive session, Ivy Bridge node Line 2  For interactive session, Ivy Bridge node
2  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
3  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email
4  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
5  qsub -I -q long -l select=300:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M email  
6    #############################
7    # 90x90x19023 configuration
8    
9    qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
10    module purge
11    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
12    cd ~/llc_4320
13    cvs co MITgcm_code
14    cvs co MITgcm_contrib/llc_hires/llc_4320
15    cd MITgcm
16    mkdir build run
17    lfs setstripe -c -1 run
18    cd build
19    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
20    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
21    emacs readtile_mpiio.c
22        tileSizeX = 90;
23        tileSizeY = 90;
24    ../tools/genmake2 -of \
25     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
26     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
27    make depend
28    make -j 16
29    
30    cd ~/llc_4320/MITgcm/run
31    cp ../build/mitgcmuv mitgcmuv_90x90x19023
32    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
33    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
34    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
35    mv data.exch2_90x90x19023 data.exch2
36    emacs data
37    
38    export MPI_BUFS_PER_PROC=1024
39    export MPI_REQUEST_MAX=65536
40    export MPI_GROUP_MAX=1024
41    export MPI_NUM_MEMORY_REGIONS=8
42    export MPI_UNBUFFERED_STDIO=1
43    export MPI_MEMMAP_OFF=1
44    export MPI_UD_TIMEOUT=100
45    mpiexec -n 20400 ./mitgcmuv_90x90x19023
46    
47    tail -f STDOUT.00000 | grep advcfl_W
48    
49    #############################
50    # 48x48x64670 configuration
51    
52    module purge
53    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
54    cd /nobackupp8/dmenemen/llc/llc_4320_new
55    cvs co MITgcm_code
56    cvs co MITgcm_contrib/llc_hires/llc_4320
57    cd MITgcm
58    mkdir build run
59    lfs setstripe -c -1 run
60    cd build
61    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_48x48x64670 SIZE.h
62    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
63    emacs readtile_mpiio.c
64        tileSizeX = 48;
65        tileSizeY = 48;
66    ../tools/genmake2 -of \
67     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
68     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
69    make depend
70    make -j 16
71    
72    cd ~/llc_4320/MITgcm/run
73    cp ../build/mitgcmuv mitgcmuv_48x48x64670
74    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/data.exch2_90x90x19023 data.exch2
75    
76    #############################
77    # generate 60x60 blank tiles
78    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
79    module purge
80    module load  comp-intel/2012.0.032 netcdf/4.0
81    module use -a ~kjtaylor/modulefiles
82    module load sles11sp3/mpt-2.10-nasa201311271217
83    cd ~/llc_4320/MITgcm
84    mkdir run_60x60
85    lfs setstripe -c -1 run_60x60
86    cd build
87    rm *
88    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
89    emacs readtile_mpiio.c
90        tileSizeX = 60;
91        tileSizeY = 60;
92    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
93    emacs SIZE.h
94         &           sNx =  60,
95         &           sNy =  60,
96         &           nSx =   6,
97         &           nPx = 11232,
98         &           Nr  =  2 )
99    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
100    emacs eeboot_minimal.F
101    C         standardMessageUnit=errorMessageUnit
102             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
103             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
104    ../tools/genmake2 -of \
105     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
106     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
107    make depend
108    make -j 16
109    cd ~/llc_4320/MITgcm/run_60x60
110    cp ../build/mitgcmuv .
111    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
112    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
113    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
114    mv data.exch2_120x120x10901 data.exch2
115    emacs data
116     tRef =  18.89, 18.89,
117     sRef =  34.84, 34.84,
118     endtime=20.,
119     delR =   1.00,    1.14,
120    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
121    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
122    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
123    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
124    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
125    emacs data.exch2
126    # remove the blank tile list
127    export MPI_BUFS_PER_PROC=1024
128    export MPI_REQUEST_MAX=65536
129    export MPI_GROUP_MAX=1024
130    export MPI_NUM_MEMORY_REGIONS=8
131    export MPI_UNBUFFERED_STDIO=1
132    export MPI_MEMMAP_OFF=1
133    export MPI_UD_TIMEOUT=100
134    mpiexec -n 12000 ./mitgcmuv
135    
136    #############################
137    # generate 45x45 blank tiles
138    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
139    module purge
140    module load  comp-intel/2012.0.032 netcdf/4.0
141    module use -a ~kjtaylor/modulefiles
142    module load sles11sp3/mpt-2.10-nasa201311271217
143    cd ~/llc_4320/MITgcm
144    mkdir run_45x45
145    lfs setstripe -c -1 run_45x45
146    cd build
147    rm *
148    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
149    emacs readtile_mpiio.c
150        tileSizeX = 45;
151        tileSizeY = 45;
152    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
153    emacs SIZE.h
154         &           sNx =  45,
155         &           sNy =  45,
156         &           nSx =  12,
157         &           nPx = 9984,
158         &           Nr  =  2 )
159    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
160    emacs eeboot_minimal.F
161    C         standardMessageUnit=errorMessageUnit
162             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
163             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
164    ../tools/genmake2 -of \
165     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
166     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
167    make depend
168    make -j 16
169    cd ~/llc_4320/MITgcm/run_45x45
170    cp ../build/mitgcmuv .
171    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
172    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
173    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
174    mv data.exch2_120x120x10901 data.exch2
175    emacs data
176     tRef =  18.89, 18.89,
177     sRef =  34.84, 34.84,
178     endtime=20.,
179     delR =   1.00,    1.14,
180    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
181    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
182    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
183    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
184    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
185    emacs data.exch2
186    # remove the blank tile list
187    export MPI_BUFS_PER_PROC=1024
188    export MPI_REQUEST_MAX=65536
189    export MPI_GROUP_MAX=1024
190    export MPI_NUM_MEMORY_REGIONS=8
191    export MPI_UNBUFFERED_STDIO=1
192    export MPI_MEMMAP_OFF=1
193    export MPI_UD_TIMEOUT=100
194    mpiexec -n 12000 ./mitgcmuv
195    
196    #############################
197    # generate 48x48 blank tiles
198    qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com
199    module purge
200    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
201    cd ~/llc_4320/MITgcm
202    mkdir run_48x48
203    lfs setstripe -c -1 run_48x48
204    cd build
205    rm *
206    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
207    emacs readtile_mpiio.c
208        tileSizeX = 48;
209        tileSizeY = 48;
210    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
211    emacs SIZE.h
212         &           sNx =  48,
213         &           sNy =  48,
214         &           nSx =  10,
215         &           nPx = 10530,
216         &           Nr  =  2 )
217    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
218    emacs eeboot_minimal.F
219    C         standardMessageUnit=errorMessageUnit
220             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
221             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
222    ../tools/genmake2 -of \
223     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
224     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
225    make depend
226    make -j 16
227    cd ~/llc_4320/MITgcm/run_48x48
228    cp ../build/mitgcmuv .
229    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
230    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
231    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
232    mv data.exch2_120x120x10901 data.exch2
233    emacs data.exch2
234    # remove blankList
235    emacs data
236     tRef =  18.89, 18.89,
237     sRef =  34.84, 34.84,
238     endtime=20.,
239     delR =   1.00,    1.14,
240    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
241    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
242    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
243    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
244    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
245    export MPI_BUFS_PER_PROC=1024
246    export MPI_REQUEST_MAX=65536
247    export MPI_GROUP_MAX=1024
248    export MPI_NUM_MEMORY_REGIONS=8
249    export MPI_UNBUFFERED_STDIO=1
250    export MPI_MEMMAP_OFF=1
251    export MPI_UD_TIMEOUT=100
252    mpiexec -n 12000 ./mitgcmuv
253    
254    #############################
255    # 120x120x10901 configuration
256    
257    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
258    module purge
259    module load  comp-intel/2012.0.032 netcdf/4.0
260    module use -a ~kjtaylor/modulefiles
261    module load sles11sp3/mpt-2.10-nasa201311271217
262    cd ~/llc_4320
263    cvs co MITgcm_code
264    cvs co MITgcm_contrib/llc_hires/llc_4320
265    cd MITgcm
266    mkdir build run
267    lfs setstripe -c -1 run
268    cd build
269    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
270    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
271    emacs readtile_mpiio.c
272        tileSizeX = 120;
273        tileSizeY = 120;
274    ../tools/genmake2 -of \
275     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
276     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
277    make depend
278    make -j 16
279    
280    cd ~/llc_4320/MITgcm/run
281    cp ../build/mitgcmuv mitgcmuv_120x120x10901
282    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
283    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
284    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
285    mv data.exch2_120x120x10901 data.exch2
286    emacs data
287    
288    export MPI_BUFS_PER_PROC=1024
289    export MPI_REQUEST_MAX=65536
290    export MPI_GROUP_MAX=1024
291    export MPI_NUM_MEMORY_REGIONS=8
292    export MPI_UNBUFFERED_STDIO=1
293    export MPI_MEMMAP_OFF=1
294    export MPI_UD_TIMEOUT=100
295    mpiexec -n 12000 ./mitgcmuv_120x120x10901
296    
297    tail -f STDOUT.00000 | grep advcfl_W
298    
299    ==============
300    
301    qsub -I -q R3089666 -l select=1750:model=ivy:aoe=sles11,walltime=04:00:00
302    tcsh
303    cd ~/llc_4320/MITgcm
304    mkdir run
305    lfs setstripe -c -1 run
306    cd run
307    cat $PBS_NODEFILE | awk '{for (i=0;i<20;++i) print $0}' > mynodes
308    setenv PBS_NODEFILE mynodes
309    cp /nobackupp8/chenze/run/mitgcmuv_72x72x29297 .
310    ln -sf /nobackupp8/chenze/run/pickup_seaice_0000000360.meta pickup_seaice.0000000180.meta
311    ln -sf /nobackupp8/chenze/run/pickup_seaice_0000000360.data pickup_seaice.0000000180.data
312    ln -sf /nobackupp8/chenze/run/pickup_0000000360.meta pickup.0000000180.meta
313    ln -sf /nobackupp8/chenze/run/pickup_0000000360.data pickup.0000000180.data
314    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
315    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
316    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
317    mv data.exch2_72x72x29297 data.exch2
318    module purge
319    module load  comp-intel/2012.0.032 netcdf/4.0
320    module use -a ~kjtaylor/modulefiles
321    module load sles11sp3/mpt-2.10-nasa201311271217
322    setenv MPI_BUFS_PER_PROC 512
323    setenv MPI_REQUEST_MAX 65536
324    setenv MPI_GROUP_MAX 1024
325    setenv MPI_NUM_MEMORY_REGIONS 8
326    setenv MPI_UNBUFFERED_STDIO 1
327    setenv MPI_MEMMAP_OFF 1
328    export MPI_UD_TIMEOUT=100
329    
330    mpiexec -n 35000 ./mitgcmuv_72x72x29297
331    
332    tail -f STDOUT.00000 | grep advcfl_w
333    
334  ==============  ==============
335    
# Line 31  mv data.exch2_72x72x29297 data.exch2 Line 358  mv data.exch2_72x72x29297 data.exch2
358  export MPI_NUM_MEMORY_REGIONS=256  export MPI_NUM_MEMORY_REGIONS=256
359  mpiexec -n 35000 ./mitgcmuv_72x72x29297  mpiexec -n 35000 ./mitgcmuv_72x72x29297
360    
361    tail -f STDOUT.00000 | grep advcfl_w
362    
363  ==============  ==============
364    
365  cd ~/llc_4320  cd ~/llc_4320
Legend:
Removed from v.1.9  
changed lines
  Added in v.1.13
  ViewVC Help
Powered by ViewVC 1.1.22