/[MITgcm]/MITgcm_contrib/llc_hires/llc_4320/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_4320/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.13 by dimitri, Fri Jun 27 13:56:32 2014 UTC revision 1.16 by dimitri, Thu Apr 14 03:51:44 2016 UTC
# Line 1  Line 1 
 For interactive session, Ivy Bridge nodes:  
 qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email  
 qsub -I -q normal -l select=300:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M email  
 qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email  
   
1  #############################  #############################
2  # 90x90x19023 configuration  # 90x90x19023 configuration
   
 qsub -I -q long -l select=1020:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  
3  module purge  module purge
4  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0  module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
5  cd ~/llc_4320  cd ~/llc_4320
6  cvs co MITgcm_code  cvs co MITgcm_code
7  cvs co MITgcm_contrib/llc_hires/llc_4320  cvs co MITgcm_contrib/llc_hires/llc_4320
8  cd MITgcm  cd MITgcm
9  mkdir build run  mkdir build run485568
10  lfs setstripe -c -1 run  lfs setstripe -c -1 run_485568
11  cd build  cd build
12    rm *
13  cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h  cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_90x90x19023 SIZE.h
14  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .  cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .
15  emacs readtile_mpiio.c  emacs readtile_mpiio.c
# Line 27  emacs readtile_mpiio.c Line 21  emacs readtile_mpiio.c
21  make depend  make depend
22  make -j 16  make -j 16
23    
24  cd ~/llc_4320/MITgcm/run  cd ~/llc_4320/MITgcm/run_485568
25  cp ../build/mitgcmuv mitgcmuv_90x90x19023  cp ../build/mitgcmuv mitgcmuv_90x90x19023_intel.2015.0.090
26  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .
27  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
28  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .
 mv data.exch2_90x90x19023 data.exch2  
29  emacs data  emacs data
30     nIter0=485568,
31    ln -sf ../run/pickup_0000485568.data pickup.0000485568.data
32    ln -sf ../run/pickup_0000485568.meta pickup.0000485568.meta
33    ln -sf ../run/pickup_seaice_0000485568.data pickup_seaice.0000485568.data
34    ln -sf ../run/pickup_seaice_0000485568.meta pickup_seaice.0000485568.meta
35    
36    qsub -I -q long -l select=850:ncpus=24:model=has,walltime=120:00:00 -m abe -M menemenlis@me.com
37    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
38    cd ~/llc_4320/MITgcm/run_485568
39    cp data.exch2_90x90x19023 data.exch2
40    mpiexec -n 20400 ./mitgcmuv_90x90x19023_intel.2015.0.090
41    tail -f STDOUT.00000 | grep advcfl_W
42    
 export MPI_BUFS_PER_PROC=1024  
 export MPI_REQUEST_MAX=65536  
 export MPI_GROUP_MAX=1024  
 export MPI_NUM_MEMORY_REGIONS=8  
 export MPI_UNBUFFERED_STDIO=1  
 export MPI_MEMMAP_OFF=1  
 export MPI_UD_TIMEOUT=100  
 mpiexec -n 20400 ./mitgcmuv_90x90x19023  
43    
44    #############################
45    # 120x120x10901 configuration
46    module purge
47    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
48    cd ~/llc_4320/MITgcm/build
49    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h
50    emacs readtile_mpiio.c
51        tileSizeX = 120;
52        tileSizeY = 120;
53    make -j 16
54    cd ~/llc_4320/MITgcm/run_485568
55    cp ../build/mitgcmuv mitgcmuv_120x120x10901_intel.2015.0.090
56    
57    qsub -I -q testing_free -l select=428:ncpus=28:model=bro,walltime=8:00:00 -m abe -M menemenlis@me.com
58    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
59    cd ~/llc_4320/MITgcm/run_485568
60    cp data.exch2_120x120x10901 data.exch2
61    mpiexec -n 11984 ./mitgcmuv_120x120x10901_intel.2015.0.090
62  tail -f STDOUT.00000 | grep advcfl_W  tail -f STDOUT.00000 | grep advcfl_W
63    
64    
65    #############################
66    # 180x180x5015 configuration
67    module purge
68    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
69    cd ~/llc_4320/MITgcm/build
70    cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_180x180x5015 SIZE.h
71    emacs readtile_mpiio.c
72        tileSizeX = 180;
73        tileSizeY = 180;
74    make -j 16
75    cd ~/llc_4320/MITgcm/run_485568
76    cp ../build/mitgcmuv mitgcmuv_180x180x5015_intel.2015.0.090
77    
78    qsub -I -q testing_free -l select=214:ncpus=28:model=bro,walltime=8:00:00 -m abe -M menemenlis@me.com
79    module load comp-intel/2015.0.090 mpi-sgi/mpt.2.12r23 netcdf/4.0
80    cd ~/llc_4320/MITgcm/run_485568
81    cp data.exch2_180x180x5015 data.exch2
82    mpiexec -n 5992 ./mitgcmuv_180x180x5015_intel.2015.0.090
83    tail -f STDOUT.00000 | grep advcfl_W
84    
85    
86  #############################  #############################
87  # 48x48x64670 configuration  # 48x48x64670 configuration
88    
89  module purge  module purge
90  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
91  cd /nobackupp8/dmenemen/llc/llc_4320_new  cd ~/llc_4320
92  cvs co MITgcm_code  cvs co MITgcm_code
93  cvs co MITgcm_contrib/llc_hires/llc_4320  cvs co MITgcm_contrib/llc_hires/llc_4320
94  cd MITgcm  cd MITgcm
# Line 73  cd ~/llc_4320/MITgcm/run Line 110  cd ~/llc_4320/MITgcm/run
110  cp ../build/mitgcmuv mitgcmuv_48x48x64670  cp ../build/mitgcmuv mitgcmuv_48x48x64670
111  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/data.exch2_90x90x19023 data.exch2  cp ../../MITgcm_contrib/llc_hires/llc_4320/input/data.exch2_90x90x19023 data.exch2
112    
113    
114  #############################  #############################
115  # generate 60x60 blank tiles  # generate 60x60 blank tiles
116  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
# Line 133  export MPI_MEMMAP_OFF=1 Line 171  export MPI_MEMMAP_OFF=1
171  export MPI_UD_TIMEOUT=100  export MPI_UD_TIMEOUT=100
172  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
173    
174    
175  #############################  #############################
176  # generate 45x45 blank tiles  # generate 45x45 blank tiles
177  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com
# Line 193  export MPI_MEMMAP_OFF=1 Line 232  export MPI_MEMMAP_OFF=1
232  export MPI_UD_TIMEOUT=100  export MPI_UD_TIMEOUT=100
233  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
234    
235    
236  #############################  #############################
237  # generate 48x48 blank tiles  # generate 48x48 blank tiles
238  qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com  qsub -I -q devel -l select=600:ncpus=20:model=ivy,walltime=2:00:00 -m abe -M menemenlis@me.com
# Line 251  export MPI_MEMMAP_OFF=1 Line 291  export MPI_MEMMAP_OFF=1
291  export MPI_UD_TIMEOUT=100  export MPI_UD_TIMEOUT=100
292  mpiexec -n 12000 ./mitgcmuv  mpiexec -n 12000 ./mitgcmuv
293    
 #############################  
 # 120x120x10901 configuration  
   
 qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M menemenlis@me.com  
 module purge  
 module load  comp-intel/2012.0.032 netcdf/4.0  
 module use -a ~kjtaylor/modulefiles  
 module load sles11sp3/mpt-2.10-nasa201311271217  
 cd ~/llc_4320  
 cvs co MITgcm_code  
 cvs co MITgcm_contrib/llc_hires/llc_4320  
 cd MITgcm  
 mkdir build run  
 lfs setstripe -c -1 run  
 cd build  
 cp ../../MITgcm_contrib/llc_hires/llc_4320/code/SIZE.h_120x120x10901 SIZE.h  
 cp ../../MITgcm_contrib/llc_hires/llc_4320/code-async/readtile_mpiio.c .  
 emacs readtile_mpiio.c  
     tileSizeX = 120;  
     tileSizeY = 120;  
 ../tools/genmake2 -of \  
  ../../MITgcm_contrib/llc_hires/llc_4320/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \  
  '../../MITgcm_contrib/llc_hires/llc_4320/code ../../MITgcm_contrib/llc_hires/llc_4320/code-async'  
 make depend  
 make -j 16  
   
 cd ~/llc_4320/MITgcm/run  
 cp ../build/mitgcmuv mitgcmuv_120x120x10901  
 ln -sf /nobackup/dmenemen/tarballs/llc_4320/run_template/* .  
 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  
 cp ../../MITgcm_contrib/llc_hires/llc_4320/input/* .  
 mv data.exch2_120x120x10901 data.exch2  
 emacs data  
   
 export MPI_BUFS_PER_PROC=1024  
 export MPI_REQUEST_MAX=65536  
 export MPI_GROUP_MAX=1024  
 export MPI_NUM_MEMORY_REGIONS=8  
 export MPI_UNBUFFERED_STDIO=1  
 export MPI_MEMMAP_OFF=1  
 export MPI_UD_TIMEOUT=100  
 mpiexec -n 12000 ./mitgcmuv_120x120x10901  
   
 tail -f STDOUT.00000 | grep advcfl_W  
294    
295  ==============  ==============
   
296  qsub -I -q R3089666 -l select=1750:model=ivy:aoe=sles11,walltime=04:00:00  qsub -I -q R3089666 -l select=1750:model=ivy:aoe=sles11,walltime=04:00:00
297  tcsh  tcsh
298  cd ~/llc_4320/MITgcm  cd ~/llc_4320/MITgcm
# Line 331  mpiexec -n 35000 ./mitgcmuv_72x72x29297 Line 326  mpiexec -n 35000 ./mitgcmuv_72x72x29297
326    
327  tail -f STDOUT.00000 | grep advcfl_w  tail -f STDOUT.00000 | grep advcfl_w
328    
 ==============  
329    
330    ==============
331  cd ~/llc_4320  cd ~/llc_4320
332  cvs co MITgcm_code  cvs co MITgcm_code
333  cvs co MITgcm_contrib/llc_hires/llc_4320  cvs co MITgcm_contrib/llc_hires/llc_4320
# Line 361  mpiexec -n 35000 ./mitgcmuv_72x72x29297 Line 356  mpiexec -n 35000 ./mitgcmuv_72x72x29297
356  tail -f STDOUT.00000 | grep advcfl_w  tail -f STDOUT.00000 | grep advcfl_w
357    
358  ==============  ==============
   
359  cd ~/llc_4320  cd ~/llc_4320
360  cvs co MITgcm_code  cvs co MITgcm_code
361  cvs co MITgcm_contrib/llc_hires/llc_4320  cvs co MITgcm_contrib/llc_hires/llc_4320
# Line 388  export MPI_NUM_MEMORY_REGIONS=256 Line 382  export MPI_NUM_MEMORY_REGIONS=256
382  mpiexec -n 6000 ./mitgcmuv_180x180x5015  mpiexec -n 6000 ./mitgcmuv_180x180x5015
383    
384  ==============  ==============
   
385  look at output  look at output
386    
387  for ts=[0 120 600:10:980 1080:120:2280]  for ts=[0 120 600:10:980 1080:120:2280]
# Line 398  for ts=[0 120 600:10:980 1080:120:2280] Line 391  for ts=[0 120 600:10:980 1080:120:2280]
391      pause(.1)      pause(.1)
392  end  end
393    
 ==============  
394    
395    ==============
396  to determine empty tiles:  to determine empty tiles:
397  grep Empty STDOUT.*  grep Empty STDOUT.*
398    
399  =============  =============
   
400  memory requirements:  memory requirements:
401  nPx  sNx sNy nSx cpu node0        total  nPx  sNx sNy nSx cpu node0        total
402  3744 180 180   2 san 22,106,128kb 5,195,641,224kb - node ran out of memory and crashed  3744 180 180   2 san 22,106,128kb 5,195,641,224kb - node ran out of memory and crashed
# Line 412  nPx  sNx sNy nSx cpu node0        total Line 404  nPx  sNx sNy nSx cpu node0        total
404  7488 180 180   1 san  7488 180 180   1 san
405    
406  =============  =============
   
407  2               =    2  2               =    2
408  3               =    3  3               =    3
409  2*2             =    4  2*2             =    4
Legend:
Removed from v.1.13  
changed lines
  Added in v.1.16
  ViewVC Help
Powered by ViewVC 1.1.22