/[MITgcm]/MITgcm_contrib/llc_hires/llc_4320/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_4320/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.12 by dimitri, Mon Feb 10 15:34:38 2014 UTC revision 1.18 by dimitri, Thu May 12 01:21:57 2016 UTC
# Line 1  Line 1 
 For interactive session, Ivy Bridge nodes:  
 qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email  
 qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email  
 qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email  
   
1  #############################  #############################
2  # 90x90x19023 configuration  # 90x90x19023 configuration
   
 qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  
3  module purge  module purge
4  module load  comp-intel/2012.0.032 netcdf/4.0  module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
 module use -a ~kjtaylor/modulefiles  
 module load sles11sp3/mpt-2.10-nasa201311271217  
5  cd ~/llc_4320  cd ~/llc_4320
6  cvs co MITgcm_code  cvs co MITgcm_code
7  cvs co MITgcm_contrib/llc_hires/llc_4320  cvs co MITgcm_contrib/llc_hires/llc_4320
8  cd MITgcm  cd MITgcm
9  mkdir build run  mkdir build run485568
10  lfs setstripe -c -1 run  lfs setstripe -c -1 run_485568
11  cd build  cd build
12    rm *
13  cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h  cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
14  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
15  emacs readtile_mpiio.c  emacs readtile_mpiio.c
# Line 28  emacs readtile_mpiio.c Line 20  emacs readtile_mpiio.c
20   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'   '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
21  make depend  make depend
22  make -j 16  make -j 16
23    cp mitgcmuv ../run_485568/mitgcmuv_90x90x19023_intel.2015.0.090
24    
25  cd ~/llc_4320/MITgcm/run  qsub -I -q long -l select=850:ncpus=24:model=has,walltime=120:00:00 -m abe -M menemenlis@me.com
26  cp ../build/mitgcmuv mitgcmuv_90x90x19023  qsub -I -q long -l select=1020:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M menemenlis@me.com
27    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
28  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
29  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
30  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
31  mv data.exch2_90x90x19023 data.exch2  cp data.exch2_90x90x19023 data.exch2
32    mv STDOUT.00000 STDOUT.485568
33  emacs data  emacs data
34     nIter0=485568,
35    ln -sf ../run/pickup_0000485568.data pickup.0000485568.data
36    ln -sf ../run/pickup_0000485568.meta pickup.0000485568.meta
37    ln -sf ../run/pickup_seaice_0000485568.data pickup_seaice.0000485568.data
38    ln -sf ../run/pickup_seaice_0000485568.meta pickup_seaice.0000485568.meta
39    mpiexec -n 20400 ./mitgcmuv_90x90x19023_intel.2015.0.090
40    
41  export MPI_BUFS_PER_PROC=1024  cd ~/llc_4320/MITgcm/run_485568
42  export MPI_REQUEST_MAX=65536  tail -f STDOUT.00000 | grep advcfl_W
43  export MPI_GROUP_MAX=1024  
44  export MPI_NUM_MEMORY_REGIONS=8  #############################
45  export MPI_UNBUFFERED_STDIO=1  # 120x120x10901 configuration
46  export MPI_MEMMAP_OFF=1  module purge
47  export MPI_UD_TIMEOUT=100  module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
48  mpiexec -n 20400 ./mitgcmuv_90x90x19023  cd ~/llc_4320/MITgcm/build
49    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
50    emacs readtile_mpiio.c
51        tileSizeX = 120;
52        tileSizeY = 120;
53    make -j 16
54    cp mitgcmuv ../run_485568/mitgcmuv_120x120x10901_intel.2015.0.090
55    
56    qsub -I -q testing_free -l select=428:ncpus=28:model=bro,walltime=8:00:00 -m abe -M menemenlis@me.com
57    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
58    cd ~/llc_4320/MITgcm/run_485568
59    cp data.exch2_120x120x10901 data.exch2
60    mv STDOUT.00000 STDOUT.489024
61    emacs -nw data
62     nIter0=489024,
63    ln -sf pickup_0000489024.data pickup.0000489024.data
64    ln -sf pickup_0000489024.meta pickup.0000489024.meta
65    ln -sf pickup_seaice_0000489024.data pickup_seaice.0000489024.data
66    ln -sf pickup_seaice_0000489024.meta pickup_seaice.0000489024.meta
67    mpiexec -n 11984 ./mitgcmuv_120x120x10901_intel.2015.0.090
68    
69    cd ~/llc_4320/MITgcm/run_485568
70  tail -f STDOUT.00000 | grep advcfl_W  tail -f STDOUT.00000 | grep advcfl_W
71    
72    
73    #############################
74    # 180x180x5015 configuration
75    module purge
76    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
77    cd ~/llc_4320/MITgcm/build
78    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_180x180x5015 SIZE.h
79    emacs readtile_mpiio.c
80        tileSizeX = 180;
81        tileSizeY = 180;
82    make -j 16
83    cp mitgcmuv ../run_485568/mitgcmuv_180x180x5015_intel.2015.0.090
84    
85    qsub -I -q testing_free -l select=214:ncpus=28:model=bro,walltime=8:00:00 -m abe -M menemenlis@me.com
86    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
87    cd ~/llc_4320/MITgcm/run_485568
88    cp data.exch2_180x180x5015 data.exch2
89    mv STDOUT.00000 STDOUT.489024
90    emacs -nw data
91     nIter0=489024,
92    ln -sf pickup_0000489024.data pickup.0000489024.data
93    ln -sf pickup_0000489024.meta pickup.0000489024.meta
94    ln -sf pickup_seaice_0000489024.data pickup_seaice.0000489024.data
95    ln -sf pickup_seaice_0000489024.meta pickup_seaice.0000489024.meta
96    mpiexec -n 5992 ./mitgcmuv_180x180x5015_intel.2015.0.090
97    
98    cd ~/llc_4320/MITgcm/run_485568
99    tail -f STDOUT.00000 | grep advcfl_W
100    
101    
102    #############################
103    # 72x72x29297 configuration
104    module purge
105    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
106    cd ~/llc_4320/MITgcm/build
107    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_72x72x29297 SIZE.h
108    emacs readtile_mpiio.c
109        tileSizeX = 72;
110        tileSizeY = 72;
111    make -j 16
112    cd ~/llc_4320/MITgcm/run_485568
113    cp mitgcmuv ../run_485568/mitgcmuv_72x72x29297_intel.2015.0.090
114    
115    qsub -I -q normal -l select=1333:ncpus=24:model=has,walltime=8:00:00 -m abe -M menemenlis@me.com
116    qsub -I -q normal -l select=1600:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M menemenlis@me.com
117    qsub -I -q wide -l select=1600:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M menemenlis@me.com
118    qsub -I -q ldsd -l select=1600:ncpus=20:model=ivy,walltime=240:00:00 -m abe -M menemenlis@me.com
119    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
120    cd ~/llc_4320/MITgcm/run_485568
121    cp data.exch2_72x72x29297 data.exch2
122    mv STDOUT.00000 STDOUT.1354752
123    emacs -nw data
124     nIter0=1354752,
125    #cp $PBS_NODEFILE mynodes
126    # >>> skip 1 then delete 20
127    #export PBS_NODEFILE="mynodes"
128    ln -sf pickup_0001354752.data pickup.0001354752.data
129    ln -sf pickup_0001354752.meta pickup.0001354752.meta
130    ln -sf pickup_seaice_0001354752.data pickup_seaice.0001354752.data
131    ln -sf pickup_seaice_0001354752.meta pickup_seaice.0001354752.meta
132    mpiexec -n 32000 ./mitgcmuv_72x72x29297_intel.2015.0.090
133    
134    cd ~/llc_4320/MITgcm/run_485568
135    tail -f STDOUT.00000 | grep advcfl_W
136    
137    
138    #############################
139    # 48x48x64670 configuration
140    module purge
141    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
142    cd ~/llc_4320
143    cvs co MITgcm_code
144    cvs co MITgcm_contrib/llc_hires/llc_4320
145    cd MITgcm
146    mkdir build run
147    lfs setstripe -c -1 run
148    cd build
149    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_48x48x64670 SIZE.h
150    cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
151    emacs readtile_mpiio.c
152        tileSizeX = 48;
153        tileSizeY = 48;
154    ../tools/genmake2 -of \
155     ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
156     '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'
157    make depend
158    make -j 16
159    
160    cd ~/llc_4320/MITgcm/run
161    cp ../build/mitgcmuv mitgcmuv_48x48x64670
162    cp ../../MITgcm_contrib/llc_hires/llc_4320/input/data.exch2_90x90x19023 data.exch2
163    
164    
165  #############################  #############################
166  # generate 60x60 blank tiles  # generate 60x60 blank tiles
167  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
# Line 108  export MPI_MEMMAP_OFF=1 Line 222  export MPI_MEMMAP_OFF=1
222  export MPI_UD_TIMEOUT=100  export MPI_UD_TIMEOUT=100
223  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
224    
225    
226  #############################  #############################
227  # generate 45x45 blank tiles  # generate 45x45 blank tiles
228  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
# Line 168  export MPI_MEMMAP_OFF=1 Line 283  export MPI_MEMMAP_OFF=1
283  export MPI_UD_TIMEOUT=100  export MPI_UD_TIMEOUT=100
284  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
285    
286    
287  #############################  #############################
288  # generate 48x48 blank tiles  # generate 48x48 blank tiles
289  qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com  qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com
290  module purge  module purge
291  module load  comp-intel/2012.0.032 netcdf/4.0  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
 module use -a ~kjtaylor/modulefiles  
 module load sles11sp3/mpt-2.10-nasa201311271217  
292  cd ~/llc_4320/MITgcm  cd ~/llc_4320/MITgcm
293  mkdir run_48x48  mkdir run_48x48
294  lfs setstripe -c -1 run_48x48  lfs setstripe -c -1 run_48x48
# Line 228  export MPI_MEMMAP_OFF=1 Line 342  export MPI_MEMMAP_OFF=1
342  export MPI_UD_TIMEOUT=100  export MPI_UD_TIMEOUT=100
343  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
344    
 #############################  
 # 120x120x10901 configuration  
   
 qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  
 module purge  
 module load  comp-intel/2012.0.032 netcdf/4.0  
 module use -a ~kjtaylor/modulefiles  
 module load sles11sp3/mpt-2.10-nasa201311271217  
 cd ~/llc_4320  
 cvs co MITgcm_code  
 cvs co MITgcm_contrib/llc_hires/llc_4320  
 cd MITgcm  
 mkdir build run  
 lfs setstripe -c -1 run  
 cd build  
 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h  
 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .  
 emacs readtile_mpiio.c  
     tileSizeX = 120;  
     tileSizeY = 120;  
 ../tools/genmake2 -of \  
  ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \  
  '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'  
 make depend  
 make -j 16  
   
 cd ~/llc_4320/MITgcm/run  
 cp ../build/mitgcmuv mitgcmuv_120x120x10901  
 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  
 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  
 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  
 mv data.exch2_120x120x10901 data.exch2  
 emacs data  
   
 export MPI_BUFS_PER_PROC=1024  
 export MPI_REQUEST_MAX=65536  
 export MPI_GROUP_MAX=1024  
 export MPI_NUM_MEMORY_REGIONS=8  
 export MPI_UNBUFFERED_STDIO=1  
 export MPI_MEMMAP_OFF=1  
 export MPI_UD_TIMEOUT=100  
 mpiexec -n 12000 ./mitgcmuv_120x120x10901  
   
 tail -f STDOUT.00000 | grep advcfl_W  
   
 ==============  
345    
346    #############################
347  qsub -I -q R3089666 -l select=1750:model=ivy:aoe=sles11,walltime=04:00:00  qsub -I -q R3089666 -l select=1750:model=ivy:aoe=sles11,walltime=04:00:00
348  tcsh  tcsh
349  cd ~/llc_4320/MITgcm  cd ~/llc_4320/MITgcm
# Line 308  mpiexec -n 35000 ./mitgcmuv_72x72x29297 Line 377  mpiexec -n 35000 ./mitgcmuv_72x72x29297
377    
378  tail -f STDOUT.00000 | grep advcfl_w  tail -f STDOUT.00000 | grep advcfl_w
379    
 ==============  
380    
381    #############################
382  cd ~/llc_4320  cd ~/llc_4320
383  cvs co MITgcm_code  cvs co MITgcm_code
384  cvs co MITgcm_contrib/llc_hires/llc_4320  cvs co MITgcm_contrib/llc_hires/llc_4320
# Line 337  mpiexec -n 35000 ./mitgcmuv_72x72x29297 Line 406  mpiexec -n 35000 ./mitgcmuv_72x72x29297
406    
407  tail -f STDOUT.00000 | grep advcfl_w  tail -f STDOUT.00000 | grep advcfl_w
408    
 ==============  
409    
410    #############################
411  cd ~/llc_4320  cd ~/llc_4320
412  cvs co MITgcm_code  cvs co MITgcm_code
413  cvs co MITgcm_contrib/llc_hires/llc_4320  cvs co MITgcm_contrib/llc_hires/llc_4320
# Line 364  mv data.exch2_72x72x29297 data.exch2 Line 433  mv data.exch2_72x72x29297 data.exch2
433  export MPI_NUM_MEMORY_REGIONS=256  export MPI_NUM_MEMORY_REGIONS=256
434  mpiexec -n 6000 ./mitgcmuv_180x180x5015  mpiexec -n 6000 ./mitgcmuv_180x180x5015
435    
 ==============  
   
 look at output  
436    
437    #############################
438    # look at output
439  for ts=[0 120 600:10:980 1080:120:2280]  for ts=[0 120 600:10:980 1080:120:2280]
440      fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],4320);      fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],4320);
441      clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar      clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar
# Line 375  for ts=[0 120 600:10:980 1080:120:2280] Line 443  for ts=[0 120 600:10:980 1080:120:2280]
443      pause(.1)      pause(.1)
444  end  end
445    
 ==============  
446    
447  to determine empty tiles:  #############################
448    # to determine empty tiles:
449  grep Empty STDOUT.*  grep Empty STDOUT.*
450    
 =============  
451    
452  memory requirements:  #############################
453  nPx  sNx sNy nSx cpu node0        total  # memory requirements:
454  3744 180 180   2 san 22,106,128kb 5,195,641,224kb - node ran out of memory and crashed  # nPx  sNx sNy nSx cpu node0        total
455  5616 120 120   3 san - node ran out of memory and crashed  # 3744 180 180   2 san 22,106,128kb 5,195,641,224kb - node ran out of memory and crashed
456  7488 180 180   1 san  # 5616 120 120   3 san - node ran out of memory and crashed
457    # 7488 180 180   1 san
458    
 =============  
459    
460    #############################
461  2               =    2  2               =    2
462  3               =    3  3               =    3
463  2*2             =    4  2*2             =    4
Legend:
Removed from v.1.12  
changed lines
  Added in v.1.18
  ViewVC Help
Powered by ViewVC 1.1.22