Actual source code: samgpetsc.c

  1: #include "petscksp.h"
 2:  #include src/mat/impls/aij/seq/aij.h
 3:  #include global.h
 4:  #include externc.h
 5:  #include samgfunc.h
 6:  #include petscfunc.h

  8: /*MC
  9:      PCSAMG -   SAMG + PETSc interface                                                
 10:                                                          
 11:     This interface allows e.g. to call samg as a shell preconditioner.    
 12:     samg is the new algebraic multigrid code by Klaus Stueben [1].         
 13:     Reference for SAMG                                                     
 14:     [1] K. St\"{u}ben,"Algebraic Multigrid: An Introduction for Positive  
 15:         Definite Problems with Applications", in [2], pp. 413--532,        
 16:         Academic Press, 2001.                                              
 17:     [2] U. Trottenberg, C. Oosterlee and A. Sch\"{u}ller, "Multigrid      
 18:         Methods", Academic Press, 2001.                                    
 19:     [1] is also available as                                               
 20:     [3] K. St\"{u}ben,"Algebraic Multigrid: An Introduction for Positive  
 21:         Definite Problems with Applications", Tech. Rep. 53, German       
 22:         National Research Center for Information Technology (GMD),        
 23:         Schloss Birlinhoven, D-53754 Sankt-Augustin, Germany, March 1999   
 24:     For more information on the SAMG-PETSc interface and examples of it's 
 25:     use, see                                                              
 26:     [4] D. Lahaye "Algebraic Multigrid for Time-Harmonic Magnetic Field    
 27:         Computations", PhD Thesis, KU Leuven, Belgium, December 2001.      
 28:         (www.cs.kuleuven.ac.be/~domenico)                                  
 29:                                                                           
 30:    Notes on PETSc part of this interface                                  
 31:    Information on how to set up shell preconditioners in PETSc can be     
 32:    in the PETSc documentation under preconditioners.                       

 34:    This preconditioner has not been completely organized to match the PETSc style,
 35:    see src/ksp/pc/impls/samgpetsc.c for the PC shell routines.

 37:     SAMG is a commercial product available from Klaus Stueben.
 38:  
 39:    Contributed by Domenico Lahaye (domenico.lahaye@cs.kuleuven.ac.be)   January 2001 
 40:                                                                           
 41: M*/

 43: /* ------------------------------------------------------------------- */
 46: /*.. SamgShellPCCreate - This routine creates a user-defined
 47:      preconditioner context.

 49:      Output Parameter:
 50:      shell - user-defined preconditioner context..*/

 52: int SamgShellPCCreate(SamgShellPC **shell)
 53: {
 54:    SamgShellPC *newctx;

 57:    PetscNew(SamgShellPC,&newctx);
 58:    *shell = newctx;
 59:    return 0;
 60: }

 62: /* ------------------------------------------------------------------- */
 65: /*..SamgShellPCSetUp - This routine sets up a user-defined
 66:     ramg preconditioner context.  

 68:     Input Parameters:
 69:     shell - user-defined preconditioner context
 70:     pmat  - preconditioner matrix

 72:     Output Parameter:
 73:     shell - fully set up user-defined preconditioner context

 75:     This routine calls the setup phase of RAMG..*/
 76: 
 77: int SamgShellPCSetUp(SamgShellPC *shell, Mat pmat)
 78: {
 79:    int              ierr, numnodes, numnonzero, nnz_count;
 80:    int              j, I, J, ncols_getrow, *cols_getrow, *diag;
 81:    PetscScalar      *vals_getrow;
 82:    MatInfo          info;
 83:    Mat_SeqAIJ       *aij = (Mat_SeqAIJ*)pmat->data;
 84:    /*..SAMG variables..*/
 85:    SAMG_PARAM       *samg_param;
 86:    double           *u_approx, *rhs, *Asky;
 87:    int              *ia, *ja;
 88:    /*..Primary SAMG parameters..*/
 89:    /*....System of equations....*/
 90:    int      matrix;
 91:    /*....Start solution and stopping criterium....*/
 92:    int      ifirst;
 93:    double   eps;
 94:    /*....Scalar and coupled system..*/
 95:    int      nsys=1, ndiu=1, ndip=1;
 96:    int      iscale[1], iu[1], ip[1];
 97:    /*....Approach and smoother....*/
 98:    int      nsolve;
 99:    /*....Cycling process....*/
100:    int      ncyc;
101:    /*....Repeated calls....*/
102:    int      iswtch;
103:    /*....Initial dimensioning..*/
104:    double   a_cmplx, g_cmplx, p_cmplx, w_avrge;
105:    /*....Class 1 input parameters controlling input/output..*/
106:    int      idump, iout;
107:    double   chktol;
108:    /*....Output parameters....*/
109:    double   res_in, res_out;
110:    int      ncyc_done;
111:    /*..Numbers of levels created by SAMG..*/
112:    int      levels;
113:    /*..Auxilary integer to set secondary parameters..*/
114:    int      intin;

116:    /*..Get size and number of unknowns of preconditioner matrix..*/
117:    MatGetSize(pmat, &numnodes, &numnodes);
118:    MatGetInfo(pmat,MAT_LOCAL,&info);
119:    numnonzero = int(info.nz_used);

121:    /*..Allocate memory for RAMG variables..*/
122:    PetscMalloc(numnonzero   * sizeof(double),&Asky);
123:    PetscMalloc((numnodes+1) * sizeof(int),&ia);
124:    PetscMalloc(numnonzero   * sizeof(int),&ja);
125:    PetscMalloc(numnodes     * sizeof(double),&u_approx);
126:    PetscMalloc(numnodes     * sizeof(double),&rhs);

128:    /*..Store PETSc matrix in compressed skyline format required by SAMG..*/
129:    nnz_count = 0;
130:    MatMarkDiagonal_SeqAIJ(pmat);
131:    diag = aij->diag;

133:    for (I=0;I<numnodes;I++){
134:      ia[I]           = nnz_count;

136:      /*....put in diagonal entry first....*/
137:      ja[nnz_count]   = I;
138:      Asky[nnz_count] = aij->a[diag[I]];
139:      nnz_count++;

141:      /*....put in off-diagonals....*/
142:      ncols_getrow = aij->i[I+1] - aij->i[I];
143:      vals_getrow  = aij->a + aij->i[I];
144:      cols_getrow  = aij->j + aij->i[I];
145:      for (j=0;j<ncols_getrow;j++){
146:        J = cols_getrow[j];
147:        if (J != I) {
148:          Asky[nnz_count] = vals_getrow[j];
149:          ja[nnz_count]   = J;
150:          nnz_count++;
151:        }
152:      }
153:    }
154:    ia[numnodes] = nnz_count;

156:    /*..Allocate memory for SAMG parameters..*/
157:    PetscNew(SAMG_PARAM,&samg_param);

159:    /*..Set SAMG parameters..*/
160:    SamgGetParam(samg_param);

162:    /*..Set Primary parameters..*/
163:    matrix  = 12;
164:    ifirst  = (*samg_param).IFIRST;
165:    eps     = (*samg_param).EPS;
166:    nsolve  = (*samg_param).NSOLVE;
167:    ncyc    = (*samg_param).NCYC;
168:    iswtch  = (*samg_param).ISWTCH;
169:    a_cmplx = (*samg_param).A_CMPLX;
170:    g_cmplx = (*samg_param).G_CMPLX;
171:    p_cmplx = (*samg_param).P_CMPLX;
172:    w_avrge = (*samg_param).W_AVRGE;
173:    chktol  = (*samg_param).CHKTOL;
174:    idump   = (*samg_param).IDUMP;
175:    iout    = (*samg_param).IOUT;
176:    /*..Set secondary parameters..*/
177:    SAMG_set_levelx(&(samg_param->LEVELX));
178:    SAMG_set_nptmn(&(samg_param->NPTMN));
179:    SAMG_set_ecg(&(samg_param->ECG));
180:    SAMG_set_ewt(&(samg_param->EWT));
181:    SAMG_set_ncg(&(samg_param->NCG));
182:    SAMG_set_nwt(&(samg_param->NWT));
183:    SAMG_set_etr(&(samg_param->ETR));
184:    SAMG_set_ntr(&(samg_param->NTR));
185:    SAMG_set_nrd(&(samg_param->NRD));
186:    SAMG_set_nru(&(samg_param->NRU));
187:    SAMG_set_nrc(&(samg_param->NRC));
188:    intin = 6; SAMG_set_logio(&intin);
189:    intin = 1; SAMG_set_mode_debug(&intin);
190:    intin = 0; SAMG_set_iter_pre(&intin);
191:    intin = 0; SAMG_set_np_opt(&intin);
192:    intin = 0; SAMG_set_ncgrad_default(&intin);

194:    /*..Reset ncyc such that only setup is performed. This is done by setting 
195:      the last digit of ncyc (the number of cycles performed) equal to zero. 
196:      The first digits of ncyc (related to the solve phase) become 
197:      irrelevant.
198:    ..*/
199:    ncyc   = 1000;

201:    /*..Switch arrays ia and ja to Fortran conventions..*/
202:    for (j=0;j<=numnodes;j++)
203:        ia[j]++;
204:    for (j=0;j<numnonzero;j++)
205:        ja[j]++;

207:    PetscPrintf(MPI_COMM_WORLD,"\n\n");
208:    PetscPrintf(MPI_COMM_WORLD,"******************************************\n");
209:    PetscPrintf(MPI_COMM_WORLD,"*** Start Setup SAMG code (scal. mode) ***\n");
210:    PetscPrintf(MPI_COMM_WORLD,"******************************************\n");
211:    PetscPrintf(MPI_COMM_WORLD,"\n\n");

213:    /*..Call SAMG..*/
214:    SAMG(&numnodes, &numnonzero, &nsys,
215:         ia, ja, Asky, rhs, u_approx, iu, &ndiu, ip, &ndip, &matrix,
216:         iscale, &res_in, &res_out, &ncyc_done, &ierr, &nsolve,
217:         &ifirst, &eps, &ncyc, &iswtch, &a_cmplx, &g_cmplx,
218:          &p_cmplx, &w_avrge, &chktol, &idump, &iout);

220:    PetscPrintf(MPI_COMM_WORLD,"\n\n");
221:    PetscPrintf(MPI_COMM_WORLD,"******************************************\n");
222:    PetscPrintf(MPI_COMM_WORLD,"*** End Setup SAMG code (scal. mode)   ***\n");
223:    PetscPrintf(MPI_COMM_WORLD,"******************************************\n");
224:    PetscPrintf(MPI_COMM_WORLD,"\n\n");

226:    /*..Get number of levels created..*/
227:    SAMGPETSC_get_levels(&levels);

229:    /*..Store RAMG output in PETSc context..*/
230:    shell->A        = Asky;
231:    shell->IA       = ia;
232:    shell->JA       = ja;
233:    shell->PARAM    = samg_param;
234:    (*shell).LEVELS = levels;

236:    return 0;
237: }

239: /* ------------------------------------------------------------------- */
242: /*..SamgShellPCApply - This routine applies the AMG code as preconditioner 

244:     Input Parameters:
245:     ctx - user-defined context, as set by SamgShellSetApply()
246:     x - input vector

248:     Output Parameter:
249:     y - preconditioned vector

251:    Notes:
252:    Note that the PCSHELL preconditioner passes a void pointer as the
253:    first input argument.  This can be cast to be the whatever the user
254:    has set (via PCSetShellApply()) the application-defined context to be.

256:    ..*/
257: /*..To apply AMG as a preconditioner we set:                        */
258: /*        i) rhs-vector equal to the residual                       */
259: /*       ii) start solution equal to zero                           */
260: /*  Implementation notes:                                           */
261: /*  For the residual (vector r) we take the values from the vector  */
262: /*  using VecGetArray. No explicit memory allocation for            */
263: /*  vals_getarray is thus  needed as VecGetArray takes care of it.  */
264: /*  The allocated memory is freed again using a call to             */
265: /*  VecRestoreArray. The values in vals_getarray are then copy to   */
266: /*  rhs of the AMG code using memcpy.                               */

268: int SamgShellPCApply(void *ctx, Vec r, Vec z)
269: {
270:    int              ierr, I, numnodes, numnonzero, *cols;
271:    SamgShellPC      *shell = (SamgShellPC *) ctx;
272:    double           *u_approx, *rhs, *Asky, *vals_getarray;
273:    int              *ia, *ja;
274:    SAMG_PARAM       *samg_param;
275:    /*..Primary SAMG parameters..*/
276:    /*....System of equations....*/
277:    int              matrix;
278:    /*....Start solution and stopping criterium....*/
279:    int              ifirst;
280:    double           eps;
281:    /*....Scalar and coupled system..*/
282:    int              nsys=1, ndiu=1, ndip=1;
283:    int              iscale[1], iu[1], ip[1];
284:    /*....Approach and smoother....*/
285:    int              nsolve;
286:    /*....Cycling process....*/
287:    int              ncyc;
288:    /*....Repeated calls....*/
289:    int              iswtch;
290:    /*....Initial dimensioning..*/
291:    double           a_cmplx, g_cmplx, p_cmplx, w_avrge;
292:    /*....Class 1 input parameters controlling input/output..*/
293:    int              idump, iout;
294:    double           chktol;
295:    /*....Output parameters....*/
296:    double           res_in, res_out;
297:    int              ncyc_done;

299:    /*..Get values from context..*/
300:    Asky       = shell->A;
301:    ia         = shell->IA;
302:    ja         = shell->JA;
303:    samg_param = shell->PARAM;

305:    /*..Get numnodes and numnonzeros..*/
306:    /*....numnodes can be determined as the size of the input vector r....*/
307:    VecGetSize(r,&numnodes);
308:    /*....numnonzero is determined from the pointer ia....*/
309:    /*....Remember that ia following Fortran conventions....*/
310:    numnonzero = ia[numnodes]-1;

312:    /*..Set the rhs of the call to ramg equal to the residual..*/
313:    VecGetArray(r,&vals_getarray);

315:    /*..Allocate memory for rhs and initial solution of call to samg..*/
316:    PetscMalloc(numnodes     * sizeof(double),&u_approx);
317:    PetscMalloc(numnodes     * sizeof(double),&rhs);

319:    /*..Set rhs of call to ramg..*/
320:    memcpy(rhs, vals_getarray, numnodes * sizeof(*rhs));
321: 
322:    /*..Set initial solution of call to ramg to zero..*/
323:    for (I=0;I<numnodes;I++){
324:        u_approx[I] = 0.;
325:    }

327:    /*..Set Primary parameters..*/
328:    matrix  = (*samg_param).MATRIX;
329:    ifirst  = (*samg_param).IFIRST;
330:    eps     = (*samg_param).EPS;
331:    nsolve  = (*samg_param).NSOLVE;
332:    ncyc    = (*samg_param).NCYC;
333:    iswtch  = (*samg_param).ISWTCH;
334:    a_cmplx = (*samg_param).A_CMPLX;
335:    g_cmplx = (*samg_param).G_CMPLX;
336:    p_cmplx = (*samg_param).P_CMPLX;
337:    w_avrge = (*samg_param).W_AVRGE;
338:    chktol  = (*samg_param).CHKTOL;
339:    idump   = (*samg_param).IDUMP;
340:    iout    = (*samg_param).IOUT;

342:    /*..Redefine iswtch to bypass setup..*/
343:    /*....First digit of iswtch = 2: bypass setup and do not release memory
344:          upon return....*/
345:    /*....Second digit of iswtch = 1: memory extension switch....*/
346:    /*....Third and fourth digit of iswtch: n_default. If n_default = 0, 
347:      the user has to set secondary parameters....*/
348:    iswtch = 210;

350:    /*..Call SAMG..*/
351:    SAMG(&numnodes, &numnonzero, &nsys,
352:         ia, ja, Asky, rhs, u_approx, iu, &ndiu, ip, &ndip, &matrix,
353:         iscale, &res_in, &res_out, &ncyc_done, &ierr, &nsolve,
354:         &ifirst, &eps, &ncyc, &iswtch, &a_cmplx, &g_cmplx,
355:         &p_cmplx, &w_avrge, &chktol, &idump, &iout);

357:    /*..Create auxilary integer array..*/
358:    PetscMalloc(numnodes * sizeof(double),&cols);

360:    for (I=0;I<numnodes;I++)
361:        cols[I] = I;

363:    /*..Store values computed by SAMG into the PETSc vector z..*/
364:    VecSetValues(z,numnodes,cols,u_approx,INSERT_VALUES);
365: 

367:    /*..Restore PETSc rhs vector..*/
368:    VecRestoreArray(r, &vals_getarray);

370:    PetscFree(cols);
371:    PetscFree(rhs);
372:    PetscFree(u_approx);

374:    return 0;
375: }

377: /* ------------------------------------------------------------------- */
380: /*..RamgShellPCDestroy - This routine destroys a user-defined
381:     preconditioner context.

383:     Input Parameter:
384:     shell - user-defined preconditioner context..*/

386: int SamgShellPCDestroy(SamgShellPC *shell)
387: {
388:   /*..Free memory allocated by samg..*/
389:   SAMG_cleanup();
390:   /*..Free PCShell context..*/
391:   PetscFree(shell->A);
392:   PetscFree(shell->IA);
393:   PetscFree(shell->JA);
394:   PetscFree(shell->PARAM);
395:   PetscFree(shell);

397:   return 0;
398: }
399: /* ------------------------------------------------------------------- */
402: /*..SamgGetParam - Gets SAMG parameters specified at runtime 
403:     OUTPUT: The parameters set in the SAMG_PARAM context
404: ..*/

406: int SamgGetParam(SAMG_PARAM *samg_param)
407: {
408:   int       ierr;

410:   /*..Set default SAMG paramets..*/
411:   /*....Class 0 SAMG parameters....*/
412:   (*samg_param).MATRIX    = 12;
413:   /*....Primary SAMG parameters....*/
414:   /*......If ifirst=0, the vector u, as passed to SAMG, is taken as first 
415:           approximation......*/
416:   (*samg_param).IFIRST    = 0;
417:   (*samg_param).EPS       = 1e-12;
418:   /*......nsolve =1 denotes scalar approach......*/
419:   (*samg_param).NSOLVE    = 1;
420:   /*......note: in the AMG-PETSc interface the number of cycles is required 
421:           to equal one to assure that in the PCApply routine AMG only performs 
422:           one cycle......*/
423:   (*samg_param).NCYC      = 1001;
424:   (*samg_param).ISWTCH    = 410;
425:   (*samg_param).A_CMPLX   = 2.5;
426:   (*samg_param).G_CMPLX   = 1.9;
427:   (*samg_param).P_CMPLX   = 1.9;
428:   (*samg_param).W_AVRGE   = 2.5;
429:   (*samg_param).CHKTOL    = 1e-8;
430:   (*samg_param).IDUMP     = 0;
431:   (*samg_param).IOUT      = 2;
432:   /*....Secundary SAMG parameters....*/
433:   (*samg_param).LEVELX    = 25;
434:   (*samg_param).NPTMN     = 100;
435:   (*samg_param).ECG       = 0.25;
436:   (*samg_param).EWT       = 0.5;
437:   (*samg_param).NCG       = 1000;
438:   (*samg_param).NWT       = 3000;
439:   (*samg_param).ETR       = 12.2;
440:   (*samg_param).NTR       = 2;
441:   (*samg_param).NRD       = 131;
442:   (*samg_param).NRC       = 0;
443:   (*samg_param).NRU       = -131;

445:   /*..Overwrite default values by values specified at runtime..*/
446:   /*....Primary SAMG parameters....*/
447:   PetscOptionsGetInt(PETSC_NULL,"-pc_samg_iswtch",&(*samg_param).ISWTCH,
448:                        PETSC_NULL);

450:   PetscOptionsGetInt(PETSC_NULL,"-pc_samg_ncyc",&(*samg_param).NCYC,
451:                        PETSC_NULL);

453:   PetscOptionsGetInt(PETSC_NULL,"-pc_samg_iout",&(*samg_param).IOUT,
454:                        PETSC_NULL);

456:  /*....Secundary SAMG parameters....*/
457:   PetscOptionsGetInt(PETSC_NULL,"-pc_samg_levelx",&(*samg_param).LEVELX,
458:                        PETSC_NULL);

460:   PetscOptionsGetInt(PETSC_NULL,"-pc_samg_nptmn",&(*samg_param).NPTMN,
461:                        PETSC_NULL);
462:   PetscOptionsGetInt(PETSC_NULL,"-pc_samg_nrd",&(*samg_param).NRD,
463:                        PETSC_NULL);

465:   PetscOptionsGetInt(PETSC_NULL,"-pc_samg_nrc",&(*samg_param).NRC,
466:                        PETSC_NULL);

468:   PetscOptionsGetInt(PETSC_NULL,"-pc_samg_nru",&(*samg_param).NRU,
469:                        PETSC_NULL);

471:   return 0;
472: }