dgoldberg/code_cg3d_petsc/cg3d_petsc_numerate.F

C $Header: /u/gcmpack/MITgcm_contrib/dgoldberg/code_cg3d_petsc/cg3d_petsc_numerate.F,v 1.2 2016/07/02 17:48:26 dgoldberg Exp $
C $Name:  $

#include "CPP_OPTIONS.h"

C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-|--+----|

CBOP
      SUBROUTINE CG3D_PETSC_NUMERATE (myThid)

C     /============================================================\
C     | SUBROUTINE                                                 |
C     | o                                                          |
C     |============================================================|
C     |                                                            |
C     \============================================================/
      IMPLICIT NONE

C     === Global variables ===
#include "SIZE.h"
#include "GRID.h"
!#include "SURFACE.h"
#include "EEPARAMS.h"
#include "PARAMS.h"
!#ifdef ALLOW_PETSC
!#include "CG3D_PETSC.h"
! UNCOMMENT IF V3.0
!#include "finclude/petscvec.h"
!#include "finclude/petscmat.h"
!#include "finclude/petscksp.h"
!#include "finclude/petscpc.h"
!#endif
#include "CG3D.h"
#ifdef ALLOW_USE_MPI
#include "EESUPPORT.h"
#endif

      INTEGER myThid

#ifdef ALLOW_NONHYDROSTATIC

      INTEGER i, j, bi, bj, k, l
      CHARACTER*(MAX_LEN_MBUF) msgBuf
#ifdef ALLOW_USE_MPI
      integer mpiRC, mpiMyWid
#endif
#ifdef ALLOW_PETSC
      _RS  DoFCountColor (MAX_CG3D_PETSC_CPUINVERT)
      _RS  DoFCountColorCumSum (MAX_CG3D_PETSC_CPUINVERT)
!      _RS  DoFCount
      integer color, rank
      integer cg3d_dofs_proc_loc (0:nPx*nPy*MAX_CG3D_PETSC_CPUINVERT-1)
      integer cg3d_dofs_cum_sum (0:nPx*nPy*MAX_CG3D_PETSC_CPUINVERT-1)
      integer dofPerRank

#ifdef ALLOW_USE_MPI
      CALL MPI_COMM_RANK( MPI_COMM_WORLD, mpiMyWId, mpiRC )
#endif

      dofPerRank = Nr/cg3d_petsc_cpuInVert

      IF (cg3d_petsc_cpuInVert.eq.1) THEN
       cg3d_color_rank(1) = mpimywid
      ELSE
! NEEDS TO CHANGE, SUCH THAT 
! CG3D_COLOR_RANK(i) = rank corresponding to ith vertical tile
! and then pushed to all other processors IN VERTICAL COLUMN ONLY
       cg3d_color_rank (1) = mpimywid
      ENDIF

      DO bj = myByLo(myThid), myByHi(myThid)
       DO bi = myBxLo(myThid), myBxHi(myThid)
        DO j=1,sNy
         DO i=1,sNx
          DO k=1,Nr
           cg3d_petsc_dofs (i,j,k,bi,bj) = -2.0
           cg3d_petsc_color (i,j,k,bi,bj) = 1 + ((k-1) / dofPerRank)
          ENDDO
         ENDDO
        ENDDO
       ENDDO
      ENDDO
 
      DO k=1, MAX_CG3D_PETSC_CPUINVERT
       IF (k.le.cg3d_petsc_cpuInVert) THEN
        DoFCountColor (k) = -1.0
       ENDIF
      ENDDO
      !DoFCount = -1.0
      DO bj = myByLo(myThid), myByHi(myThid)
       DO bi = myBxLo(myThid), myBxHi(myThid)
        DO j=1,sNy
         DO i=1,sNx
          DO k=1,Nr


C   DOFS ARE NUMBERED AS FOLLOWS ON PROCESSOR DOMAIN:
C    grid is stepped through in order bj, bi, j, i
C    1) if hfacc0(i,j,k,bi,bj)>0, the counter is updated by 1;
C        cg3d_petsc_dofs is assigned the counter;
C        o/w  cg3d_petsc_dofs is assigned -1
C    NOTE THESE NUMBERING ARRAYS ARE USED TO CONSTRUCT PETSC VECTORS AND MATRIX

          color = cg3d_petsc_color (i,j,k,bi,bj)

          if (_hFacC(i,j,k,bi,bj).gt.0.0) THEN
           DoFCountColor(color) = DoFCountColor(color) + 1.0
           cg3d_petsc_dofs (i,j,k,bi,bj) = DoFCountColor(color)
          else
           cg3d_petsc_dofs (i,j,k,bi,bj) = -1.0
          endif

          ENDDO
         ENDDO
        ENDDO
       ENDDO
      ENDDO

! SUM UP ALL DOF COUNTS WITHIN A COLUMN -- NOT GLOBAL YET
! SAME FOR ALL VERTICAL TILES IN A COLUMN
!      DoFCountColorCumSum (1) = 0.0
!      DO k=2, MAX_CG3D_PETSC_CPUINVERT
!       IF (k.le.cg3d_petsc_cpuInVert) THEN
!        DoFCountColorCumSum (k) = DoFCountColorCumSum (k-1) + 
!     &    DoFCountColor (k-1)
!       ENDIF
!      ENDDO


#ifdef ALLOW_USE_MPI

      DO i=0,nPx*nPy*MAX_CG3D_PETSC_CPUINVERT-1
#       IF (i.le.cg3d_petsc_cpuInVert) THEN
        cg3d_dofs_proc_loc (i) = 0
#       ENDIF
      ENDDO
      
      DO l=1,MAX_CG3D_PETSC_CPUINVERT
       IF (l.le.cg3d_petsc_cpuInVert) THEN
        IF (cg3d_color_rank(l).eq.mpiMyWid) THEN
         cg3d_dofs_proc_loc (mpiMyWId) = INT(DoFCountColor(l))+1
        ENDIF
       ENDIF
      ENDDO

      CALL MPI_Allreduce(cg3d_dofs_proc_loc,cg3d_dofs_process,
     &       nPx*nPy*MAX_CG3D_PETSC_CPUINVERT,MPI_INTEGER, 
     &       MPI_SUM,MPI_COMM_MODEL,mpiRC)

      cg3d_dofs_cum_sum(0) = 0

      DO i=1,nPx*nPy*MAX_CG3D_PETSC_CPUINVERT-1
        if (i.ge.1) then
!       IF (i.le.cg3d_petsc_cpuInVert) THEN
         cg3d_dofs_cum_sum(i) = cg3d_dofs_cum_sum(i-1)+
     &                     cg3d_dofs_process(i-1)
!       ENDIF
        endif
      ENDDO

#else /* ALLOW_USE_MPI */

      cg3d_dofs_process (0) = INT(DoFCountColor(1))+1
      cg3d_dofs_cum_sum (0) = INT(DoFCountColor(1))+1

#endif /* ALLOW_USE_MPI */

      DO bj = myByLo(myThid), myByHi(myThid)
       DO bi = myBxLo(myThid), myBxHi(myThid)
        DO j=1,sNy
         DO i=1,sNx
          DO k=1,Nr
           IF (cg3d_petsc_dofs(i,j,k,bi,bj).ge.0 ) THEN
           color = cg3d_petsc_color (i,j,k,bi,bj)
           rank = cg3d_color_rank (color) 
           cg3d_petsc_dofs(i,j,k,bi,bj) =
     &      cg3d_petsc_dofs(i,j,k,bi,bj) +
     &      cg3d_dofs_cum_sum(rank)
           ENDIF
          ENDDO
         ENDDO
        ENDDO
       ENDDO
      ENDDO

      _EXCH_XYZ_RS(cg3d_petsc_dofs,myThid)

      call write_fld_xyz_rs ('petscDofs1','',cg3d_petsc_dofs,0,mythid)
      call write_fld_xyz_rs ('petscColor1','',cg3d_petsc_color,0,mythid)
      print *, "GOT HERE CG3D RANK COLOR", cg3d_dofs_proc_loc

#endif /* ALLOW_PETSC */


#endif
      RETURN
      END
1	dgoldberg	1.3	C $Header: /u/gcmpack/MITgcm_contrib/dgoldberg/code_cg3d_petsc/cg3d_petsc_numerate.F,v 1.2 2016/07/02 17:48:26 dgoldberg Exp $
2	dgoldberg	1.1	C $Name: $
3
4			#include "CPP_OPTIONS.h"
5
6			C---+----1----+----2----+----3----+----4----+----5----+----6----+----7-\|--+----\|
7
8			CBOP
9			SUBROUTINE CG3D_PETSC_NUMERATE (myThid)
10
11			C /============================================================\
12			C \| SUBROUTINE \|
13			C \| o \|
14			C \|============================================================\|
15			C \| \|
16			C \============================================================/
17			IMPLICIT NONE
18
19			C === Global variables ===
20			#include "SIZE.h"
21			#include "GRID.h"
22			!#include "SURFACE.h"
23			#include "EEPARAMS.h"
24			#include "PARAMS.h"
25			!#ifdef ALLOW_PETSC
26			!#include "CG3D_PETSC.h"
27			! UNCOMMENT IF V3.0
28			!#include "finclude/petscvec.h"
29			!#include "finclude/petscmat.h"
30			!#include "finclude/petscksp.h"
31			!#include "finclude/petscpc.h"
32			!#endif
33			#include "CG3D.h"
34			#ifdef ALLOW_USE_MPI
35			#include "EESUPPORT.h"
36			#endif
37
38			INTEGER myThid
39
40			#ifdef ALLOW_NONHYDROSTATIC
41
42			INTEGER i, j, bi, bj, k, l
43			CHARACTER*(MAX_LEN_MBUF) msgBuf
44			#ifdef ALLOW_USE_MPI
45			integer mpiRC, mpiMyWid
46			#endif
47			#ifdef ALLOW_PETSC
48			_RS DoFCountColor (MAX_CG3D_PETSC_CPUINVERT)
49			_RS DoFCountColorCumSum (MAX_CG3D_PETSC_CPUINVERT)
50			! _RS DoFCount
51			integer color, rank
52			integer cg3d_dofs_proc_loc (0:nPxnPyMAX_CG3D_PETSC_CPUINVERT-1)
53			integer cg3d_dofs_cum_sum (0:nPxnPyMAX_CG3D_PETSC_CPUINVERT-1)
54			integer dofPerRank
55
56			#ifdef ALLOW_USE_MPI
57			CALL MPI_COMM_RANK( MPI_COMM_WORLD, mpiMyWId, mpiRC )
58			#endif
59
60			dofPerRank = Nr/cg3d_petsc_cpuInVert
61
62	dgoldberg	1.3	IF (cg3d_petsc_cpuInVert.eq.1) THEN
63			cg3d_color_rank(1) = mpimywid
64			ELSE
65			! NEEDS TO CHANGE, SUCH THAT
66			! CG3D_COLOR_RANK(i) = rank corresponding to ith vertical tile
67			! and then pushed to all other processors IN VERTICAL COLUMN ONLY
68			cg3d_color_rank (1) = mpimywid
69			ENDIF
70	dgoldberg	1.1
71			DO bj = myByLo(myThid), myByHi(myThid)
72			DO bi = myBxLo(myThid), myBxHi(myThid)
73			DO j=1,sNy
74			DO i=1,sNx
75			DO k=1,Nr
76			cg3d_petsc_dofs (i,j,k,bi,bj) = -2.0
77	dgoldberg	1.2	cg3d_petsc_color (i,j,k,bi,bj) = 1 + ((k-1) / dofPerRank)
78	dgoldberg	1.1	ENDDO
79			ENDDO
80			ENDDO
81			ENDDO
82			ENDDO
83
84			DO k=1, MAX_CG3D_PETSC_CPUINVERT
85			IF (k.le.cg3d_petsc_cpuInVert) THEN
86			DoFCountColor (k) = -1.0
87			ENDIF
88			ENDDO
89			!DoFCount = -1.0
90			DO bj = myByLo(myThid), myByHi(myThid)
91			DO bi = myBxLo(myThid), myBxHi(myThid)
92			DO j=1,sNy
93			DO i=1,sNx
94			DO k=1,Nr
95
96
97			C DOFS ARE NUMBERED AS FOLLOWS ON PROCESSOR DOMAIN:
98			C grid is stepped through in order bj, bi, j, i
99	dgoldberg	1.3	C 1) if hfacc0(i,j,k,bi,bj)>0, the counter is updated by 1;
100	dgoldberg	1.1	C cg3d_petsc_dofs is assigned the counter;
101			C o/w cg3d_petsc_dofs is assigned -1
102			C NOTE THESE NUMBERING ARRAYS ARE USED TO CONSTRUCT PETSC VECTORS AND MATRIX
103
104			color = cg3d_petsc_color (i,j,k,bi,bj)
105
106			if (_hFacC(i,j,k,bi,bj).gt.0.0) THEN
107			DoFCountColor(color) = DoFCountColor(color) + 1.0
108			cg3d_petsc_dofs (i,j,k,bi,bj) = DoFCountColor(color)
109			else
110			cg3d_petsc_dofs (i,j,k,bi,bj) = -1.0
111			endif
112
113			ENDDO
114			ENDDO
115			ENDDO
116			ENDDO
117			ENDDO
118
119	dgoldberg	1.3	! SUM UP ALL DOF COUNTS WITHIN A COLUMN -- NOT GLOBAL YET
120			! SAME FOR ALL VERTICAL TILES IN A COLUMN
121			! DoFCountColorCumSum (1) = 0.0
122			! DO k=2, MAX_CG3D_PETSC_CPUINVERT
123			! IF (k.le.cg3d_petsc_cpuInVert) THEN
124			! DoFCountColorCumSum (k) = DoFCountColorCumSum (k-1) +
125			! & DoFCountColor (k-1)
126			! ENDIF
127			! ENDDO
128	dgoldberg	1.1
129
130			#ifdef ALLOW_USE_MPI
131
132			DO i=0,nPxnPyMAX_CG3D_PETSC_CPUINVERT-1
133	dgoldberg	1.3	# IF (i.le.cg3d_petsc_cpuInVert) THEN
134	dgoldberg	1.1	cg3d_dofs_proc_loc (i) = 0
135	dgoldberg	1.3	# ENDIF
136	dgoldberg	1.1	ENDDO
137
138			DO l=1,MAX_CG3D_PETSC_CPUINVERT
139			IF (l.le.cg3d_petsc_cpuInVert) THEN
140			IF (cg3d_color_rank(l).eq.mpiMyWid) THEN
141			cg3d_dofs_proc_loc (mpiMyWId) = INT(DoFCountColor(l))+1
142			ENDIF
143			ENDIF
144			ENDDO
145
146			CALL MPI_Allreduce(cg3d_dofs_proc_loc,cg3d_dofs_process,
147			& nPxnPyMAX_CG3D_PETSC_CPUINVERT,MPI_INTEGER,
148			& MPI_SUM,MPI_COMM_MODEL,mpiRC)
149
150			cg3d_dofs_cum_sum(0) = 0
151
152			DO i=1,nPxnPyMAX_CG3D_PETSC_CPUINVERT-1
153	dgoldberg	1.3	if (i.ge.1) then
154			! IF (i.le.cg3d_petsc_cpuInVert) THEN
155			cg3d_dofs_cum_sum(i) = cg3d_dofs_cum_sum(i-1)+
156	dgoldberg	1.1	& cg3d_dofs_process(i-1)
157	dgoldberg	1.3	! ENDIF
158			endif
159	dgoldberg	1.1	ENDDO
160
161			#else /* ALLOW_USE_MPI */
162
163			cg3d_dofs_process (0) = INT(DoFCountColor(1))+1
164			cg3d_dofs_cum_sum (0) = INT(DoFCountColor(1))+1
165
166			#endif /* ALLOW_USE_MPI */
167
168			DO bj = myByLo(myThid), myByHi(myThid)
169			DO bi = myBxLo(myThid), myBxHi(myThid)
170			DO j=1,sNy
171			DO i=1,sNx
172			DO k=1,Nr
173			IF (cg3d_petsc_dofs(i,j,k,bi,bj).ge.0 ) THEN
174			color = cg3d_petsc_color (i,j,k,bi,bj)
175			rank = cg3d_color_rank (color)
176			cg3d_petsc_dofs(i,j,k,bi,bj) =
177			& cg3d_petsc_dofs(i,j,k,bi,bj) +
178			& cg3d_dofs_cum_sum(rank)
179			ENDIF
180			ENDDO
181			ENDDO
182			ENDDO
183			ENDDO
184			ENDDO
185
186			_EXCH_XYZ_RS(cg3d_petsc_dofs,myThid)
187
188	dgoldberg	1.2	call write_fld_xyz_rs ('petscDofs1','',cg3d_petsc_dofs,0,mythid)
189			call write_fld_xyz_rs ('petscColor1','',cg3d_petsc_color,0,mythid)
190	dgoldberg	1.3	print *, "GOT HERE CG3D RANK COLOR", cg3d_dofs_proc_loc
191	dgoldberg	1.1
192			#endif /* ALLOW_PETSC */
193
194
195			#endif
196			RETURN
197			END