Actual source code: samgmgpetsc.c
1: #include "global.h"
2: #include petscfunc.h
3: #include petscksp.h
4: #include petscmg.h
6: static char help[] = "Does PETSc multigrid cycling using hierarchy build by SAMG\n\n";
8: int samgmgpetsc(const int numnodes, double* Asky, int* ia,
9: int* ja, double* rhs, double* u_approx,
10: const OPTIONS *options)
11: {
12: /*..Petsc variables..*/
13: Vec x, b; /* approx solution and RHS */
14: Mat A; /* linear system matrix */
15: KSP ksp; /* linear solver context */
16: PC pc; /* preconditioner context */
17: PCType pctype; /* preconditioning technique */
18: KSP ksp; /* KSP context */
19: int ierr, its; /* Error messages and number of iterations */
20: /*..Other variables for the PETSc interface..*/
21: int *nnz_per_row; /* integer vector to hold the number of nonzeros */
22: /* of each row. This vector will be used to */
23: /* allocate memory for the matrix, and to store */
24: /* elements in the matrix */
25: int *cols; /* cols is a vector of collumn indices used in */
26: /* assembling the PETSc rhs vector */
27: PetscScalar *sol_array;/* sol_array used to pass the PETSc solution */
28: /* back to the calling program */
29: /*..Variables used to customize the convergence criterium to */
30: /* ||res|| / ||b|| < tol */
31: double bnrm2;
32: CONVHIST *convhist;
33: /*..Context for the SAMG preconditioner..*/
34: SamgShellPC *samg_ctx;
35: /*..Variables to extract SAMG hierarchy..*/
36: int k, levels, numnonzero;
37: double normdiff;
38: GridCtx grid[MAX_LEVELS];
39: char pathfilename[80], basefilename[80];
40: /*..Variables for intermediate levels..*/
41: KSP ksp_pre, ksp_post;
42: PC pc_pre, pc_post;
43: Mat FineLevelMatrix;
44: int petsc_level, size;
45: /*..Variables for coarse grid solve..*/
46: KSP coarsegridksp;
47: PC coarsegridpc;
48: KSP coarsegridksp;
49: int coarsegrid_n;
50: double coarsegrid_rnorm;
51: /*..Variables that determine behaviour of the code..*/
52: static int debug = *(options->DEBUG);
53: /*..Other variables..*/
54: int I;
55: PetscTruth flg, issamg, issamg_print;
56: /*..Variables for CPU timings..*/
57: PetscLogDouble v1,v2,t_setup, t_solve;
59: /*..Executable statements..*/
60: PetscInitialize( (int *) 0, (char ***) 0,(char *) 0, help);
62: /*..Get start time of linear system setup..*/
63: PetscGetTime(&v1);
65: PetscMalloc(numnodes * sizeof(int),&nnz_per_row);
67: /*..The numbero f nonzeros entries in row I can be calculated as
68: ia[I+1] - 1 - ia[I] + 1 = ia[I+1] - ia[I] ..*/
69: for (I=0;I<numnodes;I++)
70: nnz_per_row[I] = ia[I+1] - ia[I];
72: /*..Allocate (create) SeqAIJ matrix for use within PETSc..*/
73: MatCreate(PETSC_COMM_WORLD,numnodes,numnodes,numnodes,numnodes,&A);
74: MatSetType(A,MATSEQAIJ);
75: MatSeqAIJSetPreallocation(A,0,nnz_per_row);
77: /*..Assemble matrix for use within PETSc..*/
78: for (I=0;I<numnodes;I++){
79: MatSetValues(A,
80: 1, /* number of rows */
81: &I, /* pointer to global row number */
82: nnz_per_row[I], /* number of collums = number of nonzero ... */
83: /* entries in row I */
84: &(ja[ ia[I] ]),
85: /* vector global column indices */
86: (PetscScalar *) &(Asky[ ia[I] ]),
87: /* vector of coefficients */
88: INSERT_VALUES);
89: }
91: MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
92: MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
93: if (debug)
94: MatView(A,PETSC_VIEWER_STDOUT_SELF);
96: /*..Create solution and rhs vector. Note that we form vector from
97: scratch and then duplicate as needed..*/
98: VecCreate(PETSC_COMM_WORLD,&x);
99: VecSetSizes(x,PETSC_DECIDE,numnodes);
100: VecSetType(x,VECSEQ);
101: VecDuplicate(x,&b);
103: PetscMalloc(numnodes * sizeof(int),&cols);
104: for (I=0;I<numnodes;I++)
105: cols[I] = I;
107: /*..Assemble the right-hand side vector for use within PETSc..*/
108: VecSetValues(b,numnodes,cols,(PetscScalar*)rhs,INSERT_VALUES);
109:
110: VecAssemblyBegin(b);
111: VecAssemblyEnd(b);
112: if (debug){
113: printf("[PETSc]:The right-hand side \n");
114: VecView(b,PETSC_VIEWER_STDOUT_SELF);
115: printf("\n");
116: }
117: VecNorm(b,NORM_2,&bnrm2);
119: /*..Assemble the start solution vector for use within PETSc..*/
120: VecSetValues(x,numnodes,cols,(PetscScalar*)u_approx,INSERT_VALUES);
121:
122: VecAssemblyBegin(x);
123: VecAssemblyEnd(x);
125: VecNorm(b,NORM_2,&bnrm2);
126: if (debug)
127: printf("[PETSc]:The right-hand side norm = %e \n",bnrm2);
129: /*..Create linear solver context..*/
130: KSPCreate(MPI_COMM_WORLD,&ksp);
132: /*..Set operators. Here the matrix that defines the linear system
133: also serves as the preconditioning matrix..*/
134: KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);
136: /*..Extract pc type from context..*/
137: KSPGetPC(ksp,&pc);
139: /*..Customize tolerances..*/
140: KSPSetTolerances(ksp,1e-12,1e-14,PETSC_DEFAULT,
141: PETSC_DEFAULT);
143: /*..Create user defined context for the shell preconditioner..*/
144: SamgShellPCCreate(&samg_ctx);
146: /*..Do the setup for the SAMG precondioner..*/
147: SamgShellPCSetUp(samg_ctx,A);
149: /*..Give user defined preconditioner a name..*/
150: PCShellSetName(pc,"SAMG (Scalar mode)");
151:
153: /*..Parse SAMG hierarchy to PETSc variables..*/
154: levels = samg_ctx->LEVELS;
155: numnonzero = ia[numnodes];
156: SamgGetGrid(levels, numnodes, numnonzero, grid, PETSC_NULL);
158: /*..Print coarser grid and interpolation operators to file..*/
159: PetscOptionsHasName(PETSC_NULL,"-samg_print",&issamg_print);
160:
161: if (issamg_print){
162: for (k=2;k<=levels;k++){
163: sprintf(pathfilename,"./");
164: sprintf(basefilename,"Pcoarsemat.%02u",k);
165: PrintMatrix(grid[k].A, pathfilename, basefilename);
166: }
167: for (k=1;k<=levels-1;k++){
168: sprintf(basefilename,"Pinterpol.%02u%02",k, k-1);
169: PrintMatrix(grid[k].Interp, pathfilename, basefilename);
170: }
171: }
172:
173: /*..Perform check on parsing..*/
174: PetscOptionsHasName(PETSC_NULL,"-samg_check",&issamg_print);
175:
176: if (issamg_print)
177: SamgCheckGalerkin(levels, A, grid, PETSC_NULL);
178:
179: /*..Set KSP solver type..*/
180: KSPSetType(ksp,KSPRICHARDSON);
181: KSPSetMonitor(ksp,KSPDefaultMonitor,PETSC_NULL, PETSC_NULL);
182:
183: /*..Set MG preconditioner..*/
184: PCSetType(pc,PCMG);
185: MGSetLevels(pc,levels, PETSC_NULL);
186: MGSetType(pc, MGMULTIPLICATIVE);
187: MGSetCycles(pc, 1);
188: MGSetNumberSmoothUp(pc,1);
189: MGSetNumberSmoothDown(pc,1);
191: /*....Set smoother, work vectors and residual calculation on each
192: level....*/
193: for (k=1;k<=levels;k++){
194: petsc_level = levels - k;
195: /*....Get pre-smoothing KSP context....*/
196: MGGetSmootherDown(pc,petsc_level,&grid[k].ksp_pre);
197:
198: MGGetSmootherUp(pc,petsc_level,&grid[k].ksp_post);
199:
200: if (k==1)
201: FineLevelMatrix = A;
202: else
203: FineLevelMatrix = grid[k].A;
204: MatGetSize(FineLevelMatrix, &size, &size);
205: VecCreate(MPI_COMM_WORLD,&grid[k].x);
206: VecSetSizes(grid[k].x,PETSC_DECIDE,size);
207: VecSetType(grid[k].x,VECSEQ);
208: VecDuplicate(grid[k].x,&grid[k].b);
209: VecDuplicate(grid[k].x,&grid[k].r);
211: /*....set ksp_pre context....*/
212: KSPSetOperators(grid[k].ksp_pre, FineLevelMatrix,
213: FineLevelMatrix, DIFFERENT_NONZERO_PATTERN);
214:
215: KSPGetPC(ksp_pre_pre,&pc_pre);
216: KSPSetType(grid[k].ksp_pre, KSPRICHARDSON);
217: KSPSetTolerances(grid[k].ksp_pre, 1e-12, 1e-50, 1e7,1);
218:
219: PCSetType(pc_pre, PCSOR);
220: PCSORSetSymmetric(pc_pre,SOR_FORWARD_SWEEP);
222: /*....set ksp_post context....*/
223: KSPSetOperators(grid[k].ksp_post, FineLevelMatrix,
224: FineLevelMatrix, DIFFERENT_NONZERO_PATTERN);
225:
226: KSPGetPC(grid[k].ksp_post,&pc_post);
227: KSPSetInitialGuessNonzero(grid[k].ksp_post, PETSC_TRUE);
228: KSPSetType(grid[k].ksp_post, KSPRICHARDSON);
229: KSPSetTolerances(grid[k].ksp_post, 1e-12, 1e-50, 1e7,1);
230:
231: PCSetType(pc_post, PCSOR);
232: PCSORSetSymmetric(pc_post,SOR_BACKWARD_SWEEP);
234: MGSetX(pc,petsc_level,grid[k].x);
235: MGSetRhs(pc,petsc_level,grid[k].b);
236: MGSetR(pc,petsc_level,grid[k].r);
237: MGSetResidual(pc,petsc_level,MGDefaultResidual,FineLevelMatrix);
238:
239: }
241: /*....Create interpolation between the levels....*/
242: for (k=1;k<=levels-1;k++){
243: petsc_level = levels - k;
244: MGSetInterpolate(pc,petsc_level,grid[k].Interp);
245: MGSetRestriction(pc,petsc_level,grid[k].Interp);
246: }
248: /*....Set coarse grid solver....*/
249: MGGetCoarseSolve(pc,&coarsegridksp);
250: KSPSetFromOptions(coarsegridksp);
251: KSPSetOperators(coarsegridksp, grid[levels].A, grid[levels].A,
252: DIFFERENT_NONZERO_PATTERN);
253: KSPGetPC(coarsegridksp,&coarsegridpc);
254: KSPSetType(coarsegridksp, KSPPREONLY);
255: PCSetType(coarsegridpc, PCLU);
257: /*..Allow above criterea to be overwritten..*/
258: KSPSetFromOptions(ksp);
260: /*..Indicate that we are going to use a non-zero initial solution..*/
261: KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
263: /*..Get end time of linear system setup..*/
264: PetscGetTime(&v2);
265: t_setup = v2 - v1;
267: /*..Get start time of linear solve..*/
268: PetscGetTime(&v1);
270: /*..Solve linear system..*/
271: KSPSetRhs(ksp,b);
272: KSPSetSolution(ksp,x);
273: KSPSolve(ksp);
274: KSPGetIterationNumber(ksp,&its);
275:
276: /*..Print number of iterations..*/
277: PetscPrintf(PETSC_COMM_WORLD,"\n** Number of iterations done = %d \n",
278: its);
280: /*..Get end time of linear solve..*/
281: PetscGetTime(&v2);
282: t_solve = v2 - v1;
284: printf("\n[PETSc]:Time spend in setup = %e \n",t_setup);
285: printf("[PETSc]:Time spend in solve = %e \n",t_solve);
286: printf("[PETSc]:Total time = %e \n\n", t_setup + t_solve);
287:
288: /*..Copy PETSc solution back..*/
289: ierr= VecGetArray(x, &sol_array);
290: for (I=0;I<numnodes;I++){
291: u_approx[I] = sol_array[I];
292: }
293: VecRestoreArray(x,&sol_array);
295: if (debug){
296: printf("[PETSc]:The solution \n");
297: VecView(x,PETSC_VIEWER_STDOUT_SELF);
298: printf("\n");
299: }
301: /*..Free work space..*/
302: SamgShellPCDestroy(samg_ctx);
303: VecDestroy(x);
304: VecDestroy(b);
305: MatDestroy(A);
306: KSPDestroy(ksp);
307: for (k=2;k<=levels;k++){
308: MatDestroy(grid[k].A);
309: }
310: for (k=1;k<=levels-1;k++){
311: MatDestroy(grid[k].Interp);
312: }
313: for (k=1;k<=levels-1;k++){
314: VecDestroy(grid[k].b);
315: VecDestroy(grid[k].x);
316: VecDestroy(grid[k].r);
317: }
319: PetscFinalize();
320:
321: return 0;
322: }