/[MITgcm]/MITgcm_contrib/llc_hires/llc_4320/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_4320/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.11 by dimitri, Thu Jan 16 06:42:00 2014 UTC revision 1.14 by dimitri, Sat Apr 2 01:27:56 2016 UTC
# Line 1  Line 1 
1    #############################
2    # 90x90x19023 configuration
3    
4    qsub -I -q alphatst -l select=850:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
5    
6    module purge
7    module load comp-intel/2015.0.090 test/mpt.2.11r8 netcdf/4.0
8    
9    cd ~/llc_4320/MITgcm/build
10    rm *
11    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
12    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
13    emacs readtile_mpiio.c
14        tileSizeX = 90;
15        tileSizeY = 90;
16    ../tools/genmake2 -of \
17     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
18     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
19    make depend
20    make -j 16
21    
22    cd ~/llc_4320/MITgcm/run
23    cp ../build/mitgcmuv mitgcmuv_90x90x19023_intel.2015.0.090
24    cp data.exch2_90x90x19023 data.exch2
25    
26    cd ~/llc_4320/MITgcm/run
27    mv STDOUT.00000 STDOUT.409536
28    emacs data
29     nIter0=409536,
30    mv pickup_0000409536.data pickup.0000409536.data
31    mv pickup_0000409536.meta pickup.0000409536.meta
32    mv pickup_seaice_0000409536.data pickup_seaice.0000409536.data
33    mv pickup_seaice_0000409536.meta pickup_seaice.0000409536.meta
34    
35    cd ~/llc_4320/MITgcm/run
36    module purge
37    module load comp-intel/2015.0.090 test/mpt.2.11r8 netcdf/4.0
38    cp data.exch2_90x90x19023 data.exch2
39    mpiexec -n 20400 ./mitgcmuv_90x90x19023_intel.2015.0.090
40    
41    tail -f STDOUT.00000 | grep advcfl_W
42    
43    
44    #############################
45    qsub -I -q alphatst -l select=850:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
46    qsub -I -q alphatst -l select=497:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
47    
48    qsub -I -q alphatst -l select=497:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
49    module purge
50    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.10r6 netcdf/4.0
51    
52    module load comp-intel/2015.0.090 test/mpt.2.11r8 netcdf/4.0
53    
54    cd ~/llc_4320/MITgcm/build
55    rm *
56    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x19023 SIZE.h
57    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
58    emacs readtile_mpiio.c
59        tileSizeX = 120;
60        tileSizeY = 120;
61    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas .
62    emacs linux_amd64_ifort+mpi_ice_nas
63        FOPTIM='-O3 -ipo -axCORE-AVX2,AVX -xSSE4.1 -ip -fp-model precise -traceback -ftz'
64    ../tools/genmake2 -of linux_amd64_ifort+mpi_ice_nas -mpi -mods \
65     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
66    make depend
67    make -j 16
68    
69    cd ~/llc_4320/MITgcm/run
70    cp ../build/mitgcmuv mitgcmuv_120x120x10901_AVX2
71    
72    mv STDOUT.00000 STDOUT.428544
73    emacs data
74     nIter0=428544,
75    mv pickup_0000428544.data pickup.0000428544.data
76    mv pickup_0000428544.meta pickup.0000428544.meta
77    mv pickup_seaice_0000428544.data pickup_seaice.0000428544.data
78    mv pickup_seaice_0000428544.meta pickup_seaice.0000428544.meta
79    
80    cd ~/llc_4320/MITgcm/run
81    module purge
82    module load comp-intel/2015.0.090 test/mpt.2.11r8 netcdf/4.0
83    cp data.exch2_120x120x10901 data.exch2
84    mpiexec -n 12000 ./mitgcmuv_120x120x10901_Avx2
85    
86    
87  For interactive session, Ivy Bridge nodes:  For interactive session, Ivy Bridge nodes:
88  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
89  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email  qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email
90  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
91    
92    #############################
93    # 90x90x19023 configuration
94    
95  qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
96    module purge
97    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
98    cd ~/llc_4320
99    cvs co MITgcm_code
100    cvs co MITgcm_contrib/llc_hires/llc_4320
101    cd MITgcm
102    mkdir build run
103    lfs setstripe -c -1 run
104    cd build
105    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
106    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
107    emacs readtile_mpiio.c
108        tileSizeX = 90;
109        tileSizeY = 90;
110    ../tools/genmake2 -of \
111     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
112     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
113    make depend
114    make -j 16
115    
116    cd ~/llc_4320/MITgcm/run
117    cp ../build/mitgcmuv mitgcmuv_90x90x19023
118    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
119    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
120    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
121    cp data.exch2_90x90x19023 data.exch2
122    emacs data
123    
124    export MPI_BUFS_PER_PROC=1024
125    export MPI_REQUEST_MAX=65536
126    export MPI_GROUP_MAX=1024
127    export MPI_NUM_MEMORY_REGIONS=8
128    export MPI_UNBUFFERED_STDIO=1
129    export MPI_MEMMAP_OFF=1
130    export MPI_UD_TIMEOUT=100
131    mpiexec -n 20400 ./mitgcmuv_90x90x19023
132    
133    tail -f STDOUT.00000 | grep advcfl_W
134    
135  #############################  #############################
136  # generate 60x60 blank tiles  # 48x48x64670 configuration
137    
138    module purge
139    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
140    cd ~/llc_4320
141    cvs co MITgcm_code
142    cvs co MITgcm_contrib/llc_hires/llc_4320
143    cd MITgcm
144    mkdir build run
145    lfs setstripe -c -1 run
146    cd build
147    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_48x48x64670 SIZE.h
148    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
149    emacs readtile_mpiio.c
150        tileSizeX = 48;
151        tileSizeY = 48;
152    ../tools/genmake2 -of \
153     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
154     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
155    make depend
156    make -j 16
157    
158    cd ~/llc_4320/MITgcm/run
159    cp ../build/mitgcmuv mitgcmuv_48x48x64670
160    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/data.exch2_90x90x19023 data.exch2
161    
162    #############################
163    # generate 60x60 blank tiles
164  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
165  module purge  module purge
166  module load  comp-intel/2012.0.032 netcdf/4.0  module load  comp-intel/2012.0.032 netcdf/4.0
# Line 15  module load sles11sp3/mpt-2.10-nasa20131 Line 169  module load sles11sp3/mpt-2.10-nasa20131
169  cd ~/llc_4320/MITgcm  cd ~/llc_4320/MITgcm
170  mkdir run_60x60  mkdir run_60x60
171  lfs setstripe -c -1 run_60x60  lfs setstripe -c -1 run_60x60
   
172  cd build  cd build
173  rm *  rm *
174  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
# Line 34  emacs eeboot_minimal.F Line 187  emacs eeboot_minimal.F
187  C         standardMessageUnit=errorMessageUnit  C         standardMessageUnit=errorMessageUnit
188           WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)           WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
189           OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')           OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
   
190  ../tools/genmake2 -of \  ../tools/genmake2 -of \
191   ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \   ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
192   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
193  make depend  make depend
194  make -j 16  make -j 16
   
195  cd ~/llc_4320/MITgcm/run_60x60  cd ~/llc_4320/MITgcm/run_60x60
196  cp ../build/mitgcmuv .  cp ../build/mitgcmuv .
197  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
198  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
199  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
200  mv data.exch2_120x120x10901 data.exch2  mv data.exch2_120x120x10901 data.exch2
   
201  emacs data  emacs data
202   tRef =  18.89, 18.89,   tRef =  18.89, 18.89,
203   sRef =  34.84, 34.84,   sRef =  34.84, 34.84,
# Line 58  emacs data Line 208  emacs data
208  # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',  # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
209  # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',  # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
210  # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',  # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
   
211  emacs data.exch2  emacs data.exch2
212  # remove the blank tile list  # remove the blank tile list
   
213  export MPI_BUFS_PER_PROC=1024  export MPI_BUFS_PER_PROC=1024
214  export MPI_REQUEST_MAX=65536  export MPI_REQUEST_MAX=65536
215  export MPI_GROUP_MAX=1024  export MPI_GROUP_MAX=1024
216  export MPI_NUM_MEMORY_REGIONS=8  export MPI_NUM_MEMORY_REGIONS=8
217  export MPI_UNBUFFERED_STDIO=1  export MPI_UNBUFFERED_STDIO=1
218  export MPI_MEMMAP_OFF=1  export MPI_MEMMAP_OFF=1
219    export MPI_UD_TIMEOUT=100
220  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
221    
222  #############################  #############################
223  # generate 45x45 blank tiles  # generate 45x45 blank tiles
   
224  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
225  module purge  module purge
226  module load  comp-intel/2012.0.032 netcdf/4.0  module load  comp-intel/2012.0.032 netcdf/4.0
# Line 81  module load sles11sp3/mpt-2.10-nasa20131 Line 229  module load sles11sp3/mpt-2.10-nasa20131
229  cd ~/llc_4320/MITgcm  cd ~/llc_4320/MITgcm
230  mkdir run_45x45  mkdir run_45x45
231  lfs setstripe -c -1 run_45x45  lfs setstripe -c -1 run_45x45
   
232  cd build  cd build
233  rm *  rm *
234  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
# Line 100  emacs eeboot_minimal.F Line 247  emacs eeboot_minimal.F
247  C         standardMessageUnit=errorMessageUnit  C         standardMessageUnit=errorMessageUnit
248           WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)           WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
249           OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')           OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
   
250  ../tools/genmake2 -of \  ../tools/genmake2 -of \
251   ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \   ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
252   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
253  make depend  make depend
254  make -j 16  make -j 16
   
255  cd ~/llc_4320/MITgcm/run_45x45  cd ~/llc_4320/MITgcm/run_45x45
256  cp ../build/mitgcmuv .  cp ../build/mitgcmuv .
257  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
258  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
259  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
260  mv data.exch2_120x120x10901 data.exch2  mv data.exch2_120x120x10901 data.exch2
   
261  emacs data  emacs data
262   tRef =  18.89, 18.89,   tRef =  18.89, 18.89,
263   sRef =  34.84, 34.84,   sRef =  34.84, 34.84,
# Line 124  emacs data Line 268  emacs data
268  # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',  # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
269  # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',  # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
270  # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',  # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
   
271  emacs data.exch2  emacs data.exch2
272  # remove the blank tile list  # remove the blank tile list
   
273  export MPI_BUFS_PER_PROC=1024  export MPI_BUFS_PER_PROC=1024
274  export MPI_REQUEST_MAX=65536  export MPI_REQUEST_MAX=65536
275  export MPI_GROUP_MAX=1024  export MPI_GROUP_MAX=1024
276  export MPI_NUM_MEMORY_REGIONS=8  export MPI_NUM_MEMORY_REGIONS=8
277  export MPI_UNBUFFERED_STDIO=1  export MPI_UNBUFFERED_STDIO=1
278  export MPI_MEMMAP_OFF=1  export MPI_MEMMAP_OFF=1
279    export MPI_UD_TIMEOUT=100
280  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
281    
282    #############################
283    # generate 48x48 blank tiles
284    qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com
285    module purge
286    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
287    cd ~/llc_4320/MITgcm
288    mkdir run_48x48
289    lfs setstripe -c -1 run_48x48
290    cd build
291    rm *
292    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
293    emacs readtile_mpiio.c
294        tileSizeX = 48;
295        tileSizeY = 48;
296    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
297    emacs SIZE.h
298         &           sNx =  48,
299         &           sNy =  48,
300         &           nSx =  10,
301         &           nPx = 10530,
302         &           Nr  =  2 )
303    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/eeboot_minimal.F .
304    emacs eeboot_minimal.F
305    C         standardMessageUnit=errorMessageUnit
306             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
307             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
308    ../tools/genmake2 -of \
309     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
310     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
311    make depend
312    make -j 16
313    cd ~/llc_4320/MITgcm/run_48x48
314    cp ../build/mitgcmuv .
315    ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
316    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
317    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
318    mv data.exch2_120x120x10901 data.exch2
319    emacs data.exch2
320    # remove blankList
321    emacs data
322     tRef =  18.89, 18.89,
323     sRef =  34.84, 34.84,
324     endtime=20.,
325     delR =   1.00,    1.14,
326    # hydrogThetaFile='THETA_llc2160_10sep2011_4320x56160x90_r4',
327    # hydrogSaltFile ='SALT_llc2160_10sep2011_4320x56160x90_r4',
328    # uVelInitFile   ='UVEL_llc2160_10sep2011_4320x56160x90_r4',
329    # vVelInitFile   ='VVEL_llc2160_10sep2011_4320x56160x90_r4',
330    # pSurfInitFile  ='ETAN_llc2160_10sep2011_4320x56160_r4',
331    export MPI_BUFS_PER_PROC=1024
332    export MPI_REQUEST_MAX=65536
333    export MPI_GROUP_MAX=1024
334    export MPI_NUM_MEMORY_REGIONS=8
335    export MPI_UNBUFFERED_STDIO=1
336    export MPI_MEMMAP_OFF=1
337    export MPI_UD_TIMEOUT=100
338    mpiexec -n 12000 ./mitgcmuv
339    
340  #############################  #############################
341  # 120x120x10901 configuration  # 120x120x10901 configuration
# Line 177  export MPI_GROUP_MAX=1024 Line 377  export MPI_GROUP_MAX=1024
377  export MPI_NUM_MEMORY_REGIONS=8  export MPI_NUM_MEMORY_REGIONS=8
378  export MPI_UNBUFFERED_STDIO=1  export MPI_UNBUFFERED_STDIO=1
379  export MPI_MEMMAP_OFF=1  export MPI_MEMMAP_OFF=1
380    export MPI_UD_TIMEOUT=100
381  mpiexec -n 12000 ./mitgcmuv_120x120x10901  mpiexec -n 12000 ./mitgcmuv_120x120x10901
382    
383  tail -f STDOUT.00000 | grep advcfl_W  tail -f STDOUT.00000 | grep advcfl_W
# Line 210  setenv MPI_GROUP_MAX 1024 Line 411  setenv MPI_GROUP_MAX 1024
411  setenv MPI_NUM_MEMORY_REGIONS 8  setenv MPI_NUM_MEMORY_REGIONS 8
412  setenv MPI_UNBUFFERED_STDIO 1  setenv MPI_UNBUFFERED_STDIO 1
413  setenv MPI_MEMMAP_OFF 1  setenv MPI_MEMMAP_OFF 1
414    export MPI_UD_TIMEOUT=100
415    
416  mpiexec -n 35000 ./mitgcmuv_72x72x29297  mpiexec -n 35000 ./mitgcmuv_72x72x29297
417    
Legend:
Removed from v.1.11  
changed lines
  Added in v.1.14
  ViewVC Help
Powered by ViewVC 1.1.22