/[MITgcm]/MITgcm_contrib/llc_hires/llc_4320/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_4320/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.11 by dimitri, Thu Jan 16 06:42:00 2014 UTC revision 1.15 by dimitri, Sat Apr 2 01:35:44 2016 UTC
# Line 1  Line 1 
1    #############################
2    # 90x90x19023 configuration
3    
4    qsub -I -q alphatst -l select=850:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
5    
6    module purge
7    module load comp-intel/2015.0.090 test/mpt.2.11r8 netcdf/4.0
8    
9    cd ~/llc_4320
10    cvs co MITgcm_code
11    cvs co MITgcm_contrib/llc_hires/llc_4320
12    cd MITgcm
13    mkdir build run
14    lfs setstripe -c -1 run
15    cd build
16    rm *
17    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
18    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
19    emacs readtile_mpiio.c
20        tileSizeX = 90;
21        tileSizeY = 90;
22    ../tools/genmake2 -of \
23     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
24     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
25    make depend
26    make -j 16
27    
28    cd ~/llc_4320/MITgcm/run
29    cp ../build/mitgcmuv mitgcmuv_90x90x19023_intel.2015.0.090
30    cp data.exch2_90x90x19023 data.exch2
31    
32    cd ~/llc_4320/MITgcm/run
33    mv STDOUT.00000 STDOUT.409536
34    emacs data
35     nIter0=409536,
36    mv pickup_0000409536.data pickup.0000409536.data
37    mv pickup_0000409536.meta pickup.0000409536.meta
38    mv pickup_seaice_0000409536.data pickup_seaice.0000409536.data
39    mv pickup_seaice_0000409536.meta pickup_seaice.0000409536.meta
40    
41    cd ~/llc_4320/MITgcm/run
42    module purge
43    module load comp-intel/2015.0.090 test/mpt.2.11r8 netcdf/4.0
44    cp data.exch2_90x90x19023 data.exch2
45    mpiexec -n 20400 ./mitgcmuv_90x90x19023_intel.2015.0.090
46    
47    tail -f STDOUT.00000 | grep advcfl_W
48    
49    
50    #############################
51    qsub -I -q alphatst -l select=850:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
52    qsub -I -q alphatst -l select=497:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
53    
54    qsub -I -q alphatst -l select=497:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
55    module purge
56    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.10r6 netcdf/4.0
57    
58    module load comp-intel/2015.0.090 test/mpt.2.11r8 netcdf/4.0
59    
60    cd ~/llc_4320/MITgcm/build
61    rm *
62    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x19023 SIZE.h
63    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
64    emacs readtile_mpiio.c
65        tileSizeX = 120;
66        tileSizeY = 120;
67    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas .
68    emacs linux_amd64_ifort+mpi_ice_nas
69        FOPTIM='-O3 -ipo -axCORE-AVX2,AVX -xSSE4.1 -ip -fp-model precise -traceback -ftz'
70    ../tools/genmake2 -of linux_amd64_ifort+mpi_ice_nas -mpi -mods \
71     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
72    make depend
73    make -j 16
74    
75    cd ~/llc_4320/MITgcm/run
76    cp ../build/mitgcmuv mitgcmuv_120x120x10901_AVX2
77    
78    mv STDOUT.00000 STDOUT.428544
79    emacs data
80     nIter0=428544,
81    mv pickup_0000428544.data pickup.0000428544.data
82    mv pickup_0000428544.meta pickup.0000428544.meta
83    mv pickup_seaice_0000428544.data pickup_seaice.0000428544.data
84    mv pickup_seaice_0000428544.meta pickup_seaice.0000428544.meta
85    
86    cd ~/llc_4320/MITgcm/run
87    module purge
88    module load comp-intel/2015.0.090 test/mpt.2.11r8 netcdf/4.0
89    cp data.exch2_120x120x10901 data.exch2
90    mpiexec -n 12000 ./mitgcmuv_120x120x10901_Avx2
91    
92    
93  For interactive session, Ivy Bridge nodes:  For interactive session, Ivy Bridge nodes:
94  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
95  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email
96  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
97    
98    #############################
99    # 90x90x19023 configuration
100    
101  qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
102    module purge
103    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
104    cd ~/llc_4320
105    cvs co MITgcm_code
106    cvs co MITgcm_contrib/llc_hires/llc_4320
107    cd MITgcm
108    mkdir build run
109    lfs setstripe -c -1 run
110    cd build
111    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
112    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
113    emacs readtile_mpiio.c
114        tileSizeX = 90;
115        tileSizeY = 90;
116    ../tools/genmake2 -of \
117     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
118     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
119    make depend
120    make -j 16
121    
122    cd ~/llc_4320/MITgcm/run
123    cp ../build/mitgcmuv mitgcmuv_90x90x19023
124    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
125    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
126    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
127    cp data.exch2_90x90x19023 data.exch2
128    emacs data
129    
130    export MPI_BUFS_PER_PROC=1024
131    export MPI_REQUEST_MAX=65536
132    export MPI_GROUP_MAX=1024
133    export MPI_NUM_MEMORY_REGIONS=8
134    export MPI_UNBUFFERED_STDIO=1
135    export MPI_MEMMAP_OFF=1
136    export MPI_UD_TIMEOUT=100
137    mpiexec -n 20400 ./mitgcmuv_90x90x19023
138    
139    tail -f STDOUT.00000 | grep advcfl_W
140    
141  #############################  #############################
142  # generate 60x60 blank tiles  # 48x48x64670 configuration
143    
144    module purge
145    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
146    cd ~/llc_4320
147    cvs co MITgcm_code
148    cvs co MITgcm_contrib/llc_hires/llc_4320
149    cd MITgcm
150    mkdir build run
151    lfs setstripe -c -1 run
152    cd build
153    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_48x48x64670 SIZE.h
154    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
155    emacs readtile_mpiio.c
156        tileSizeX = 48;
157        tileSizeY = 48;
158    ../tools/genmake2 -of \
159     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
160     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
161    make depend
162    make -j 16
163    
164    cd ~/llc_4320/MITgcm/run
165    cp ../build/mitgcmuv mitgcmuv_48x48x64670
166    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/data.exch2_90x90x19023 data.exch2
167    
168    #############################
169    # generate 60x60 blank tiles
170  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
171  module purge  module purge
172  module load  comp-intel/2012.0.032 netcdf/4.0  module load  comp-intel/2012.0.032 netcdf/4.0
# Line 15  module load sles11sp3/mpt-2.10-nasa20131 Line 175  module load sles11sp3/mpt-2.10-nasa20131
175  cd ~/llc_4320/MITgcm  cd ~/llc_4320/MITgcm
176  mkdir run_60x60  mkdir run_60x60
177  lfs setstripe -c -1 run_60x60  lfs setstripe -c -1 run_60x60
   
178  cd build  cd build
179  rm *  rm *
180  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
# Line 34  emacs eeboot_minimal.F Line 193  emacs eeboot_minimal.F
193  C         standardMessageUnit=errorMessageUnit  C         standardMessageUnit=errorMessageUnit
194           WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)           WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
195           OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')           OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
   
196  ../tools/genmake2 -of \  ../tools/genmake2 -of \
197   ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \   ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
198   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
199  make depend  make depend
200  make -j 16  make -j 16
   
201  cd ~/llc_4320/MITgcm/run_60x60  cd ~/llc_4320/MITgcm/run_60x60
202  cp ../build/mitgcmuv .  cp ../build/mitgcmuv .
203  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
204  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
205  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
206  mv data.exch2_120x120x10901 data.exch2  mv data.exch2_120x120x10901 data.exch2
   
207  emacs data  emacs data
208   tRef =  18.89, 18.89,   tRef =  18.89, 18.89,
209   sRef =  34.84, 34.84,   sRef =  34.84, 34.84,
# Line 58  emacs data Line 214  emacs data
214  # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',  # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
215  # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',  # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
216  # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',  # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
   
217  emacs data.exch2  emacs data.exch2
218  # remove the blank tile list  # remove the blank tile list
   
219  export MPI_BUFS_PER_PROC=1024  export MPI_BUFS_PER_PROC=1024
220  export MPI_REQUEST_MAX=65536  export MPI_REQUEST_MAX=65536
221  export MPI_GROUP_MAX=1024  export MPI_GROUP_MAX=1024
222  export MPI_NUM_MEMORY_REGIONS=8  export MPI_NUM_MEMORY_REGIONS=8
223  export MPI_UNBUFFERED_STDIO=1  export MPI_UNBUFFERED_STDIO=1
224  export MPI_MEMMAP_OFF=1  export MPI_MEMMAP_OFF=1
225    export MPI_UD_TIMEOUT=100
226  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
227    
228  #############################  #############################
229  # generate 45x45 blank tiles  # generate 45x45 blank tiles
   
230  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
231  module purge  module purge
232  module load  comp-intel/2012.0.032 netcdf/4.0  module load  comp-intel/2012.0.032 netcdf/4.0
# Line 81  module load sles11sp3/mpt-2.10-nasa20131 Line 235  module load sles11sp3/mpt-2.10-nasa20131
235  cd ~/llc_4320/MITgcm  cd ~/llc_4320/MITgcm
236  mkdir run_45x45  mkdir run_45x45
237  lfs setstripe -c -1 run_45x45  lfs setstripe -c -1 run_45x45
   
238  cd build  cd build
239  rm *  rm *
240  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
# Line 100  emacs eeboot_minimal.F Line 253  emacs eeboot_minimal.F
253  C         standardMessageUnit=errorMessageUnit  C         standardMessageUnit=errorMessageUnit
254           WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)           WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
255           OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')           OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
   
256  ../tools/genmake2 -of \  ../tools/genmake2 -of \
257   ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \   ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
258   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
259  make depend  make depend
260  make -j 16  make -j 16
   
261  cd ~/llc_4320/MITgcm/run_45x45  cd ~/llc_4320/MITgcm/run_45x45
262  cp ../build/mitgcmuv .  cp ../build/mitgcmuv .
263  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
264  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
265  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
266  mv data.exch2_120x120x10901 data.exch2  mv data.exch2_120x120x10901 data.exch2
   
267  emacs data  emacs data
268   tRef =  18.89, 18.89,   tRef =  18.89, 18.89,
269   sRef =  34.84, 34.84,   sRef =  34.84, 34.84,
# Line 124  emacs data Line 274  emacs data
274  # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',  # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
275  # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',  # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
276  # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',  # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
   
277  emacs data.exch2  emacs data.exch2
278  # remove the blank tile list  # remove the blank tile list
   
279  export MPI_BUFS_PER_PROC=1024  export MPI_BUFS_PER_PROC=1024
280  export MPI_REQUEST_MAX=65536  export MPI_REQUEST_MAX=65536
281  export MPI_GROUP_MAX=1024  export MPI_GROUP_MAX=1024
282  export MPI_NUM_MEMORY_REGIONS=8  export MPI_NUM_MEMORY_REGIONS=8
283  export MPI_UNBUFFERED_STDIO=1  export MPI_UNBUFFERED_STDIO=1
284  export MPI_MEMMAP_OFF=1  export MPI_MEMMAP_OFF=1
285    export MPI_UD_TIMEOUT=100
286  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
287    
288    #############################
289    # generate 48x48 blank tiles
290    qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com
291    module purge
292    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
293    cd ~/llc_4320/MITgcm
294    mkdir run_48x48
295    lfs setstripe -c -1 run_48x48
296    cd build
297    rm *
298    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
299    emacs readtile_mpiio.c
300        tileSizeX = 48;
301        tileSizeY = 48;
302    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
303    emacs SIZE.h
304         &           sNx =  48,
305         &           sNy =  48,
306         &           nSx =  10,
307         &           nPx = 10530,
308         &           Nr  =  2 )
309    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
310    emacs eeboot_minimal.F
311    C         standardMessageUnit=errorMessageUnit
312             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
313             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
314    ../tools/genmake2 -of \
315     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
316     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
317    make depend
318    make -j 16
319    cd ~/llc_4320/MITgcm/run_48x48
320    cp ../build/mitgcmuv .
321    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
322    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
323    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
324    mv data.exch2_120x120x10901 data.exch2
325    emacs data.exch2
326    # remove blankList
327    emacs data
328     tRef =  18.89, 18.89,
329     sRef =  34.84, 34.84,
330     endtime=20.,
331     delR =   1.00,    1.14,
332    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
333    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
334    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
335    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
336    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
337    export MPI_BUFS_PER_PROC=1024
338    export MPI_REQUEST_MAX=65536
339    export MPI_GROUP_MAX=1024
340    export MPI_NUM_MEMORY_REGIONS=8
341    export MPI_UNBUFFERED_STDIO=1
342    export MPI_MEMMAP_OFF=1
343    export MPI_UD_TIMEOUT=100
344    mpiexec -n 12000 ./mitgcmuv
345    
346  #############################  #############################
347  # 120x120x10901 configuration  # 120x120x10901 configuration
# Line 177  export MPI_GROUP_MAX=1024 Line 383  export MPI_GROUP_MAX=1024
383  export MPI_NUM_MEMORY_REGIONS=8  export MPI_NUM_MEMORY_REGIONS=8
384  export MPI_UNBUFFERED_STDIO=1  export MPI_UNBUFFERED_STDIO=1
385  export MPI_MEMMAP_OFF=1  export MPI_MEMMAP_OFF=1
386    export MPI_UD_TIMEOUT=100
387  mpiexec -n 12000 ./mitgcmuv_120x120x10901  mpiexec -n 12000 ./mitgcmuv_120x120x10901
388    
389  tail -f STDOUT.00000 | grep advcfl_W  tail -f STDOUT.00000 | grep advcfl_W
# Line 210  setenv MPI_GROUP_MAX 1024 Line 417  setenv MPI_GROUP_MAX 1024
417  setenv MPI_NUM_MEMORY_REGIONS 8  setenv MPI_NUM_MEMORY_REGIONS 8
418  setenv MPI_UNBUFFERED_STDIO 1  setenv MPI_UNBUFFERED_STDIO 1
419  setenv MPI_MEMMAP_OFF 1  setenv MPI_MEMMAP_OFF 1
420    export MPI_UD_TIMEOUT=100
421    
422  mpiexec -n 35000 ./mitgcmuv_72x72x29297  mpiexec -n 35000 ./mitgcmuv_72x72x29297
423    
Legend:
Removed from v.1.11  
changed lines
  Added in v.1.15
  ViewVC Help
Powered by ViewVC 1.1.22