Actual source code: samgmgpetsc.c

  1: #include "global.h"
 2:  #include petscfunc.h
 3:  #include petscksp.h
 4:  #include petscmg.h

  6: static char help[] = "Does PETSc multigrid cycling using hierarchy build by SAMG\n\n";

  8: int samgmgpetsc(const int numnodes, double* Asky, int* ia, 
  9:                 int* ja, double* rhs, double* u_approx, 
 10:                 const OPTIONS *options)
 11: {
 12:    /*..Petsc variables..*/
 13:    Vec      x, b;         /* approx solution and RHS */
 14:    Mat      A;            /* linear system matrix */
 15:    KSP     ksp;         /* linear solver context */
 16:    PC       pc;           /* preconditioner context */
 17:    PCType   pctype;       /* preconditioning technique */
 18:    KSP      ksp;          /* KSP context */
 19:    int      ierr, its;    /* Error messages and number of iterations */
 20:    /*..Other variables for the PETSc interface..*/
 21:    int      *nnz_per_row; /* integer vector to hold the number of nonzeros */
 22:                           /* of each row. This vector will be used to      */
 23:                           /* allocate memory for the matrix, and to store  */
 24:                           /* elements in the matrix                        */
 25:    int      *cols;        /* cols is a vector of collumn indices used in   */
 26:                           /* assembling the PETSc rhs vector               */
 27:    PetscScalar *sol_array;/* sol_array used to pass the PETSc solution     */
 28:                           /* back to the calling program                   */
 29:    /*..Variables used to customize the convergence criterium to            */
 30:    /*  ||res|| / ||b|| < tol                                               */
 31:    double      bnrm2;
 32:    CONVHIST    *convhist;
 33:    /*..Context for the SAMG preconditioner..*/
 34:    SamgShellPC *samg_ctx;
 35:    /*..Variables to extract SAMG hierarchy..*/
 36:    int         k, levels, numnonzero;
 37:    double      normdiff;
 38:    GridCtx     grid[MAX_LEVELS];
 39:    char        pathfilename[80], basefilename[80];
 40:    /*..Variables for intermediate levels..*/
 41:    KSP         ksp_pre, ksp_post;
 42:    PC          pc_pre, pc_post;
 43:    Mat         FineLevelMatrix;
 44:    int         petsc_level, size;
 45:    /*..Variables for coarse grid solve..*/
 46:    KSP        coarsegridksp;
 47:    PC          coarsegridpc;
 48:    KSP         coarsegridksp;
 49:    int         coarsegrid_n;
 50:    double      coarsegrid_rnorm;
 51:    /*..Variables that determine behaviour of the code..*/
 52:    static int  debug = *(options->DEBUG);
 53:    /*..Other variables..*/
 54:    int         I;
 55:    PetscTruth  flg, issamg, issamg_print;
 56:    /*..Variables for CPU timings..*/
 57:    PetscLogDouble  v1,v2,t_setup, t_solve;

 59:    /*..Executable statements..*/
 60:    PetscInitialize( (int *) 0, (char ***) 0,(char *) 0, help);

 62:    /*..Get start time of linear system setup..*/
 63:    PetscGetTime(&v1);

 65:    PetscMalloc(numnodes * sizeof(int),&nnz_per_row);

 67:    /*..The numbero f nonzeros entries in row I can be calculated as      
 68:        ia[I+1] - 1 - ia[I] + 1 = ia[I+1] - ia[I]                         ..*/
 69:    for (I=0;I<numnodes;I++)
 70:        nnz_per_row[I] = ia[I+1] - ia[I];

 72:    /*..Allocate (create) SeqAIJ matrix  for use within PETSc..*/
 73:    MatCreate(PETSC_COMM_WORLD,numnodes,numnodes,numnodes,numnodes,&A);
 74:    MatSetType(A,MATSEQAIJ);
 75:    MatSeqAIJSetPreallocation(A,0,nnz_per_row);

 77:   /*..Assemble matrix  for use within PETSc..*/
 78:    for (I=0;I<numnodes;I++){
 79:       MatSetValues(A,
 80:                1,              /* number of rows */
 81:                &I,             /* pointer to global row number */
 82:                nnz_per_row[I], /* number of collums = number of nonzero ... */
 83:                                /* entries in row I                          */
 84:                &(ja[ ia[I] ]),
 85:                               /* vector global column indices */
 86:                (PetscScalar *) &(Asky[ ia[I] ]),
 87:                               /* vector of coefficients */
 88:         INSERT_VALUES);
 89:    }

 91:    MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
 92:    MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
 93:    if (debug)
 94:        MatView(A,PETSC_VIEWER_STDOUT_SELF);

 96:    /*..Create solution and rhs vector.  Note that we form  vector from 
 97:      scratch and then duplicate as needed..*/
 98:    VecCreate(PETSC_COMM_WORLD,&x);
 99:    VecSetSizes(x,PETSC_DECIDE,numnodes);
100:    VecSetType(x,VECSEQ);
101:    VecDuplicate(x,&b);

103:    PetscMalloc(numnodes * sizeof(int),&cols);
104:    for (I=0;I<numnodes;I++)
105:        cols[I] = I;

107:    /*..Assemble the right-hand side vector for use within PETSc..*/
108:    VecSetValues(b,numnodes,cols,(PetscScalar*)rhs,INSERT_VALUES);
109: 
110:    VecAssemblyBegin(b);
111:    VecAssemblyEnd(b);
112:    if (debug){
113:       printf("[PETSc]:The right-hand side \n");
114:       VecView(b,PETSC_VIEWER_STDOUT_SELF);
115:       printf("\n");
116:    }
117:    VecNorm(b,NORM_2,&bnrm2);

119:    /*..Assemble the start solution vector for use within PETSc..*/
120:    VecSetValues(x,numnodes,cols,(PetscScalar*)u_approx,INSERT_VALUES);
121: 
122:    VecAssemblyBegin(x);
123:    VecAssemblyEnd(x);

125:    VecNorm(b,NORM_2,&bnrm2);
126:    if (debug)
127:        printf("[PETSc]:The right-hand side norm = %e \n",bnrm2);

129:    /*..Create linear solver context..*/
130:    KSPCreate(MPI_COMM_WORLD,&ksp);

132:    /*..Set operators. Here the matrix that defines the linear system
133:      also serves as the preconditioning matrix..*/
134:    KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);

136:    /*..Extract pc type from context..*/
137:    KSPGetPC(ksp,&pc);

139:    /*..Customize tolerances..*/
140:    KSPSetTolerances(ksp,1e-12,1e-14,PETSC_DEFAULT,
141:           PETSC_DEFAULT);

143:     /*..Create user defined context for the shell preconditioner..*/
144:    SamgShellPCCreate(&samg_ctx);

146:    /*..Do the setup for the SAMG precondioner..*/
147:    SamgShellPCSetUp(samg_ctx,A);

149:    /*..Give user defined preconditioner a name..*/
150:    PCShellSetName(pc,"SAMG (Scalar mode)");
151: 

153:    /*..Parse SAMG hierarchy to PETSc variables..*/
154:    levels = samg_ctx->LEVELS;
155:    numnonzero = ia[numnodes];
156:    SamgGetGrid(levels, numnodes, numnonzero, grid, PETSC_NULL);

158:    /*..Print coarser grid and interpolation operators to file..*/
159:    PetscOptionsHasName(PETSC_NULL,"-samg_print",&issamg_print);
160: 
161:    if (issamg_print){
162:      for (k=2;k<=levels;k++){
163:          sprintf(pathfilename,"./");
164:          sprintf(basefilename,"Pcoarsemat.%02u",k);
165:          PrintMatrix(grid[k].A, pathfilename, basefilename);
166:      }
167:      for (k=1;k<=levels-1;k++){
168:          sprintf(basefilename,"Pinterpol.%02u%02",k, k-1);
169:          PrintMatrix(grid[k].Interp, pathfilename, basefilename);
170:      }
171:    }
172: 
173:    /*..Perform check on parsing..*/
174:    PetscOptionsHasName(PETSC_NULL,"-samg_check",&issamg_print);
175: 
176:    if (issamg_print)
177:      SamgCheckGalerkin(levels, A, grid, PETSC_NULL);
178: 
179:    /*..Set KSP solver type..*/
180:    KSPSetType(ksp,KSPRICHARDSON);
181:    KSPSetMonitor(ksp,KSPDefaultMonitor,PETSC_NULL, PETSC_NULL);
182: 
183:    /*..Set MG preconditioner..*/
184:    PCSetType(pc,PCMG);
185:    MGSetLevels(pc,levels, PETSC_NULL);
186:    MGSetType(pc, MGMULTIPLICATIVE);
187:    MGSetCycles(pc, 1);
188:    MGSetNumberSmoothUp(pc,1);
189:    MGSetNumberSmoothDown(pc,1);

191:    /*....Set smoother, work vectors and residual calculation on each 
192:          level....*/
193:    for (k=1;k<=levels;k++){
194:        petsc_level = levels - k;
195:        /*....Get pre-smoothing KSP context....*/
196:        MGGetSmootherDown(pc,petsc_level,&grid[k].ksp_pre);
197: 
198:        MGGetSmootherUp(pc,petsc_level,&grid[k].ksp_post);
199: 
200:        if (k==1)
201:           FineLevelMatrix = A;
202:           else
203:           FineLevelMatrix = grid[k].A;
204:        MatGetSize(FineLevelMatrix, &size, &size);
205:        VecCreate(MPI_COMM_WORLD,&grid[k].x);
206:        VecSetSizes(grid[k].x,PETSC_DECIDE,size);
207:        VecSetType(grid[k].x,VECSEQ);
208:        VecDuplicate(grid[k].x,&grid[k].b);
209:        VecDuplicate(grid[k].x,&grid[k].r);

211:        /*....set ksp_pre context....*/
212:        KSPSetOperators(grid[k].ksp_pre, FineLevelMatrix,
213:                                FineLevelMatrix, DIFFERENT_NONZERO_PATTERN);
214: 
215:        KSPGetPC(ksp_pre_pre,&pc_pre);
216:        KSPSetType(grid[k].ksp_pre, KSPRICHARDSON);
217:        KSPSetTolerances(grid[k].ksp_pre, 1e-12, 1e-50, 1e7,1);
218: 
219:        PCSetType(pc_pre, PCSOR);
220:        PCSORSetSymmetric(pc_pre,SOR_FORWARD_SWEEP);

222:        /*....set ksp_post context....*/
223:        KSPSetOperators(grid[k].ksp_post, FineLevelMatrix,
224:                                FineLevelMatrix, DIFFERENT_NONZERO_PATTERN);
225: 
226:        KSPGetPC(grid[k].ksp_post,&pc_post);
227:        KSPSetInitialGuessNonzero(grid[k].ksp_post, PETSC_TRUE);
228:        KSPSetType(grid[k].ksp_post, KSPRICHARDSON);
229:        KSPSetTolerances(grid[k].ksp_post, 1e-12, 1e-50, 1e7,1);
230: 
231:        PCSetType(pc_post, PCSOR);
232:        PCSORSetSymmetric(pc_post,SOR_BACKWARD_SWEEP);

234:        MGSetX(pc,petsc_level,grid[k].x);
235:        MGSetRhs(pc,petsc_level,grid[k].b);
236:        MGSetR(pc,petsc_level,grid[k].r);
237:        MGSetResidual(pc,petsc_level,MGDefaultResidual,FineLevelMatrix);
238: 
239:    }

241:    /*....Create interpolation between the levels....*/
242:    for (k=1;k<=levels-1;k++){
243:      petsc_level = levels - k;
244:      MGSetInterpolate(pc,petsc_level,grid[k].Interp);
245:      MGSetRestriction(pc,petsc_level,grid[k].Interp);
246:    }

248:    /*....Set coarse grid solver....*/
249:    MGGetCoarseSolve(pc,&coarsegridksp);
250:    KSPSetFromOptions(coarsegridksp);
251:    KSPSetOperators(coarsegridksp, grid[levels].A, grid[levels].A,
252:                            DIFFERENT_NONZERO_PATTERN);
253:    KSPGetPC(coarsegridksp,&coarsegridpc);
254:    KSPSetType(coarsegridksp, KSPPREONLY);
255:    PCSetType(coarsegridpc, PCLU);

257:    /*..Allow above criterea to be overwritten..*/
258:    KSPSetFromOptions(ksp);

260:    /*..Indicate that we are going to use a non-zero initial solution..*/
261:    KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);

263:    /*..Get end time of linear system setup..*/
264:    PetscGetTime(&v2);
265:    t_setup = v2 - v1;

267:    /*..Get start time of linear solve..*/
268:    PetscGetTime(&v1);

270:    /*..Solve linear system..*/
271:    KSPSetRhs(ksp,b);
272:    KSPSetSolution(ksp,x);
273:    KSPSolve(ksp);
274:    KSPGetIterationNumber(ksp,&its);
275: 
276:    /*..Print number of iterations..*/
277:    PetscPrintf(PETSC_COMM_WORLD,"\n** Number of iterations done = %d \n",
278:                its);

280:    /*..Get end time of linear solve..*/
281:    PetscGetTime(&v2);
282:    t_solve = v2 - v1;

284:    printf("\n[PETSc]:Time spend in setup = %e \n",t_setup);
285:    printf("[PETSc]:Time spend in solve = %e \n",t_solve);
286:    printf("[PETSc]:Total time = %e \n\n", t_setup + t_solve);
287: 
288:    /*..Copy PETSc solution back..*/
289:    ierr= VecGetArray(x, &sol_array);
290:    for (I=0;I<numnodes;I++){
291:      u_approx[I] = sol_array[I];
292:    }
293:    VecRestoreArray(x,&sol_array);

295:    if (debug){
296:       printf("[PETSc]:The solution \n");
297:       VecView(x,PETSC_VIEWER_STDOUT_SELF);
298:       printf("\n");
299:    }

301:    /*..Free work space..*/
302:    SamgShellPCDestroy(samg_ctx);
303:    VecDestroy(x);
304:    VecDestroy(b);
305:    MatDestroy(A);
306:    KSPDestroy(ksp);
307:    for (k=2;k<=levels;k++){
308:      MatDestroy(grid[k].A);
309:    }
310:    for (k=1;k<=levels-1;k++){
311:      MatDestroy(grid[k].Interp);
312:    }
313:    for (k=1;k<=levels-1;k++){
314:      VecDestroy(grid[k].b);
315:      VecDestroy(grid[k].x);
316:      VecDestroy(grid[k].r);
317:    }

319:    PetscFinalize();
320: 
321:    return 0;
322: }