Actual source code: asm.c

  1: /*$Id: asm.c,v 1.131 2001/08/07 03:03:38 balay Exp $*/
  2: /*
  3:   This file defines an additive Schwarz preconditioner for any Mat implementation.

  5:   Note that each processor may have any number of subdomains. But in order to 
  6:   deal easily with the VecScatter(), we treat each processor as if it has the
  7:   same number of subdomains.

  9:        n - total number of true subdomains on all processors
 10:        n_local_true - actual number of subdomains on this processor
 11:        n_local = maximum over all processors of n_local_true
 12: */
 13:  #include src/ksp/pc/pcimpl.h

 15: typedef struct {
 16:   int        n,n_local,n_local_true;
 17:   PetscTruth is_flg;              /* flg set to 1 if the IS created in pcsetup */
 18:   int        overlap;             /* overlap requested by user */
 19:   KSP       *ksp;               /* linear solvers for each block */
 20:   VecScatter *scat;               /* mapping to subregion */
 21:   Vec        *x,*y;
 22:   IS         *is;                 /* index set that defines each subdomain */
 23:   Mat        *mat,*pmat;          /* mat is not currently used */
 24:   PCASMType  type;                /* use reduced interpolation, restriction or both */
 25:   PetscTruth same_local_solves;   /* flag indicating whether all local solvers are same */
 26:   PetscTruth inplace;             /* indicates that the sub-matrices are deleted after 
 27:                                      PCSetUpOnBlocks() is done. Similar to inplace 
 28:                                      factorization in the case of LU and ILU */
 29: } PC_ASM;

 33: static int PCView_ASM(PC pc,PetscViewer viewer)
 34: {
 35:   PC_ASM      *jac = (PC_ASM*)pc->data;
 36:   int         rank,ierr,i;
 37:   const char  *cstring = 0;
 38:   PetscTruth  isascii,isstring;
 39:   PetscViewer sviewer;


 43:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
 44:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
 45:   if (isascii) {
 46:     PetscViewerASCIIPrintf(viewer,"  Additive Schwarz: total subdomain blocks = %d, amount of overlap = %d\n",jac->n,jac->overlap);
 47:     if (jac->type == PC_ASM_NONE)             cstring = "limited restriction and interpolation (PC_ASM_NONE)";
 48:     else if (jac->type == PC_ASM_RESTRICT)    cstring = "full restriction (PC_ASM_RESTRICT)";
 49:     else if (jac->type == PC_ASM_INTERPOLATE) cstring = "full interpolation (PC_ASM_INTERPOLATE)";
 50:     else if (jac->type == PC_ASM_BASIC)       cstring = "full restriction and interpolation (PC_ASM_BASIC)";
 51:     else                                      cstring = "Unknown ASM type";
 52:     PetscViewerASCIIPrintf(viewer,"  Additive Schwarz: type - %s\n",cstring);
 53:     MPI_Comm_rank(pc->comm,&rank);
 54:     if (jac->same_local_solves) {
 55:       PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:\n");
 56:       PetscViewerGetSingleton(viewer,&sviewer);
 57:       if (!rank && jac->ksp) {
 58:         PetscViewerASCIIPushTab(viewer);
 59:         KSPView(jac->ksp[0],sviewer);
 60:         PetscViewerASCIIPopTab(viewer);
 61:       }
 62:       PetscViewerRestoreSingleton(viewer,&sviewer);
 63:     } else {
 64:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:\n");
 65:       PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: number of local blocks = %d\n",rank,jac->n_local);
 66:       PetscViewerASCIIPushTab(viewer);
 67:       for (i=0; i<jac->n_local; i++) {
 68:         PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: local block number %d\n",rank,i);
 69:         PetscViewerGetSingleton(viewer,&sviewer);
 70:         KSPView(jac->ksp[i],sviewer);
 71:         PetscViewerRestoreSingleton(viewer,&sviewer);
 72:         if (i != jac->n_local-1) {
 73:           PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
 74:         }
 75:       }
 76:       PetscViewerASCIIPopTab(viewer);
 77:       PetscViewerFlush(viewer);
 78:     }
 79:   } else if (isstring) {
 80:     PetscViewerStringSPrintf(viewer," blks=%d, overlap=%d, type=%d",jac->n,jac->overlap,jac->type);
 81:     PetscViewerGetSingleton(viewer,&sviewer);
 82:       if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
 83:     PetscViewerGetSingleton(viewer,&sviewer);
 84:   } else {
 85:     SETERRQ1(1,"Viewer type %s not supported for PCASM",((PetscObject)viewer)->type_name);
 86:   }
 87:   return(0);
 88: }

 92: static int PCSetUp_ASM(PC pc)
 93: {
 94:   PC_ASM   *osm  = (PC_ASM*)pc->data;
 95:   int      i,ierr,m,n_local = osm->n_local,n_local_true = osm->n_local_true;
 96:   int      start,start_val,end_val,size,sz,bs;
 97:   MatReuse scall = MAT_REUSE_MATRIX;
 98:   IS       isl;
 99:   KSP      ksp;
100:   PC       subpc;
101:   char     *prefix,*pprefix;

104:   if (!pc->setupcalled) {
105:     if (osm->n == PETSC_DECIDE && osm->n_local_true == PETSC_DECIDE) {
106:       /* no subdomains given, use one per processor */
107:       osm->n_local_true = osm->n_local = 1;
108:       MPI_Comm_size(pc->comm,&size);
109:       osm->n = size;
110:     } else if (osm->n == PETSC_DECIDE) { /* determine global number of subdomains */
111:       int inwork[2],outwork[2];
112:       inwork[0] = inwork[1] = osm->n_local_true;
113:       MPI_Allreduce(inwork,outwork,1,MPI_2INT,PetscMaxSum_Op,pc->comm);
114:       osm->n_local = outwork[0];
115:       osm->n       = outwork[1];
116:     }
117:     n_local      = osm->n_local;
118:     n_local_true = osm->n_local_true;
119:     if (!osm->is){ /* build the index sets */
120:       PetscMalloc((n_local_true+1)*sizeof(IS **),&osm->is);
121:       MatGetOwnershipRange(pc->pmat,&start_val,&end_val);
122:       MatGetBlockSize(pc->pmat,&bs);
123:       sz    = end_val - start_val;
124:       start = start_val;
125:       if (end_val/bs*bs != end_val || start_val/bs*bs != start_val) {
126:         SETERRQ(PETSC_ERR_ARG_WRONG,"Bad distribution for matrix block size");
127:       }
128:       for (i=0; i<n_local_true; i++){
129:         size       =  ((sz/bs)/n_local_true + (((sz/bs) % n_local_true) > i))*bs;
130:          ISCreateStride(PETSC_COMM_SELF,size,start,1,&isl);
131:         start      += size;
132:         osm->is[i] =  isl;
133:       }
134:       osm->is_flg = PETSC_TRUE;
135:     }

137:     PetscMalloc((n_local_true+1)*sizeof(KSP **),&osm->ksp);
138:     PetscMalloc(n_local*sizeof(VecScatter **),&osm->scat);
139:     PetscMalloc(2*n_local*sizeof(Vec **),&osm->x);
140:     osm->y = osm->x + n_local;

142:     /*  Extend the "overlapping" regions by a number of steps  */
143:     MatIncreaseOverlap(pc->pmat,n_local_true,osm->is,osm->overlap);
144:     for (i=0; i<n_local_true; i++) {
145:       ISSort(osm->is[i]);
146:     }

148:     /* create the local work vectors and scatter contexts */
149:     for (i=0; i<n_local_true; i++) {
150:       ISGetLocalSize(osm->is[i],&m);
151:       VecCreateSeq(PETSC_COMM_SELF,m,&osm->x[i]);
152:       VecDuplicate(osm->x[i],&osm->y[i]);
153:       ISCreateStride(PETSC_COMM_SELF,m,0,1,&isl);
154:       VecScatterCreate(pc->vec,osm->is[i],osm->x[i],isl,&osm->scat[i]);
155:       ISDestroy(isl);
156:     }
157:     for (i=n_local_true; i<n_local; i++) {
158:       VecCreateSeq(PETSC_COMM_SELF,0,&osm->x[i]);
159:       VecDuplicate(osm->x[i],&osm->y[i]);
160:       ISCreateStride(PETSC_COMM_SELF,0,0,1,&isl);
161:       VecScatterCreate(pc->vec,isl,osm->x[i],isl,&osm->scat[i]);
162:       ISDestroy(isl);
163:     }

165:    /* 
166:        Create the local solvers.
167:     */
168:     for (i=0; i<n_local_true; i++) {
169:       KSPCreate(PETSC_COMM_SELF,&ksp);
170:       PetscLogObjectParent(pc,ksp);
171:       KSPSetType(ksp,KSPPREONLY);
172:       KSPGetPC(ksp,&subpc);
173:       PCSetType(subpc,PCILU);
174:       PCGetOptionsPrefix(pc,&prefix);
175:       KSPSetOptionsPrefix(ksp,prefix);
176:       KSPAppendOptionsPrefix(ksp,"sub_");
177:       KSPSetFromOptions(ksp);
178:       osm->ksp[i] = ksp;
179:     }
180:     scall = MAT_INITIAL_MATRIX;
181:   } else {
182:     /* 
183:        Destroy the blocks from the previous iteration
184:     */
185:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
186:       MatDestroyMatrices(osm->n_local_true,&osm->pmat);
187:       scall = MAT_INITIAL_MATRIX;
188:     }
189:   }

191:   /* extract out the submatrices */
192:   MatGetSubMatrices(pc->pmat,osm->n_local_true,osm->is,osm->is,scall,&osm->pmat);

194:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
195:      different boundary conditions for the submatrices than for the global problem) */
196:   PCModifySubMatrices(pc,osm->n_local,osm->is,osm->is,osm->pmat,pc->modifysubmatricesP);

198:   /* loop over subdomains putting them into local ksp */
199:   PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
200:   for (i=0; i<n_local_true; i++) {
201:     PetscObjectSetOptionsPrefix((PetscObject)osm->pmat[i],pprefix);
202:     PetscLogObjectParent(pc,osm->pmat[i]);
203:     KSPSetOperators(osm->ksp[i],osm->pmat[i],osm->pmat[i],pc->flag);
204:   }
205:   return(0);
206: }

210: static int PCSetUpOnBlocks_ASM(PC pc)
211: {
212:   PC_ASM *osm = (PC_ASM*)pc->data;
213:   int    i,ierr;

216:   for (i=0; i<osm->n_local_true; i++) {
217:     KSPSetRhs(osm->ksp[i],osm->x[i]);
218:     KSPSetSolution(osm->ksp[i],osm->y[i]);
219:     KSPSetUp(osm->ksp[i]);
220:   }
221:   /* 
222:      If inplace flag is set, then destroy the matrix after the setup
223:      on blocks is done.
224:   */
225:   if (osm->inplace && osm->n_local_true > 0) {
226:     MatDestroyMatrices(osm->n_local_true,&osm->pmat);
227:   }
228:   return(0);
229: }

233: static int PCApply_ASM(PC pc,Vec x,Vec y)
234: {
235:   PC_ASM      *osm = (PC_ASM*)pc->data;
236:   int         i,n_local = osm->n_local,n_local_true = osm->n_local_true,ierr;
237:   PetscScalar zero = 0.0;
238:   ScatterMode forward = SCATTER_FORWARD,reverse = SCATTER_REVERSE;

241:   /*
242:        Support for limiting the restriction or interpolation to only local 
243:      subdomain values (leaving the other values 0). 
244:   */
245:   if (!(osm->type & PC_ASM_RESTRICT)) {
246:     forward = SCATTER_FORWARD_LOCAL;
247:     /* have to zero the work RHS since scatter may leave some slots empty */
248:     for (i=0; i<n_local; i++) {
249:       VecSet(&zero,osm->x[i]);
250:     }
251:   }
252:   if (!(osm->type & PC_ASM_INTERPOLATE)) {
253:     reverse = SCATTER_REVERSE_LOCAL;
254:   }

256:   for (i=0; i<n_local; i++) {
257:     VecScatterBegin(x,osm->x[i],INSERT_VALUES,forward,osm->scat[i]);
258:   }
259:   VecSet(&zero,y);
260:   /* do the local solves */
261:   for (i=0; i<n_local_true; i++) {
262:     VecScatterEnd(x,osm->x[i],INSERT_VALUES,forward,osm->scat[i]);
263:     KSPSetRhs(osm->ksp[i],osm->x[i]);
264:     KSPSetSolution(osm->ksp[i],osm->y[i]);
265:     KSPSolve(osm->ksp[i]);
266:     VecScatterBegin(osm->y[i],y,ADD_VALUES,reverse,osm->scat[i]);
267:   }
268:   /* handle the rest of the scatters that do not have local solves */
269:   for (i=n_local_true; i<n_local; i++) {
270:     VecScatterEnd(x,osm->x[i],INSERT_VALUES,forward,osm->scat[i]);
271:     VecScatterBegin(osm->y[i],y,ADD_VALUES,reverse,osm->scat[i]);
272:   }
273:   for (i=0; i<n_local; i++) {
274:     VecScatterEnd(osm->y[i],y,ADD_VALUES,reverse,osm->scat[i]);
275:   }
276:   return(0);
277: }

281: static int PCApplyTranspose_ASM(PC pc,Vec x,Vec y)
282: {
283:   PC_ASM      *osm = (PC_ASM*)pc->data;
284:   int         i,n_local = osm->n_local,n_local_true = osm->n_local_true,ierr;
285:   PetscScalar zero = 0.0;
286:   ScatterMode forward = SCATTER_FORWARD,reverse = SCATTER_REVERSE;

289:   /*
290:        Support for limiting the restriction or interpolation to only local 
291:      subdomain values (leaving the other values 0).

293:        Note: these are reversed from the PCApply_ASM() because we are applying the 
294:      transpose of the three terms 
295:   */
296:   if (!(osm->type & PC_ASM_INTERPOLATE)) {
297:     forward = SCATTER_FORWARD_LOCAL;
298:     /* have to zero the work RHS since scatter may leave some slots empty */
299:     for (i=0; i<n_local; i++) {
300:       VecSet(&zero,osm->x[i]);
301:     }
302:   }
303:   if (!(osm->type & PC_ASM_RESTRICT)) {
304:     reverse = SCATTER_REVERSE_LOCAL;
305:   }

307:   for (i=0; i<n_local; i++) {
308:     VecScatterBegin(x,osm->x[i],INSERT_VALUES,forward,osm->scat[i]);
309:   }
310:   VecSet(&zero,y);
311:   /* do the local solves */
312:   for (i=0; i<n_local_true; i++) {
313:     VecScatterEnd(x,osm->x[i],INSERT_VALUES,forward,osm->scat[i]);
314:     KSPSetRhs(osm->ksp[i],osm->x[i]);
315:     KSPSetSolution(osm->ksp[i],osm->y[i]);
316:     KSPSolveTranspose(osm->ksp[i]);
317:     VecScatterBegin(osm->y[i],y,ADD_VALUES,reverse,osm->scat[i]);
318:   }
319:   /* handle the rest of the scatters that do not have local solves */
320:   for (i=n_local_true; i<n_local; i++) {
321:     VecScatterEnd(x,osm->x[i],INSERT_VALUES,forward,osm->scat[i]);
322:     VecScatterBegin(osm->y[i],y,ADD_VALUES,reverse,osm->scat[i]);
323:   }
324:   for (i=0; i<n_local; i++) {
325:     VecScatterEnd(osm->y[i],y,ADD_VALUES,reverse,osm->scat[i]);
326:   }
327:   return(0);
328: }

332: static int PCDestroy_ASM(PC pc)
333: {
334:   PC_ASM *osm = (PC_ASM*)pc->data;
335:   int    i,ierr;

338:   for (i=0; i<osm->n_local; i++) {
339:     VecScatterDestroy(osm->scat[i]);
340:     VecDestroy(osm->x[i]);
341:     VecDestroy(osm->y[i]);
342:   }
343:   if (osm->n_local_true > 0 && !osm->inplace && osm->pmat) {
344:     MatDestroyMatrices(osm->n_local_true,&osm->pmat);
345:   }
346:   if (osm->ksp) {
347:     for (i=0; i<osm->n_local_true; i++) {
348:       KSPDestroy(osm->ksp[i]);
349:     }
350:   }
351:   if (osm->is_flg) {
352:     for (i=0; i<osm->n_local_true; i++) {ISDestroy(osm->is[i]);}
353:     PetscFree(osm->is);
354:   }
355:   if (osm->ksp) {PetscFree(osm->ksp);}
356:   if (osm->scat) {PetscFree(osm->scat);}
357:   if (osm->x) {PetscFree(osm->x);}
358:   PetscFree(osm);
359:   return(0);
360: }

364: static int PCSetFromOptions_ASM(PC pc)
365: {
366:   PC_ASM     *osm = (PC_ASM*)pc->data;
367:   int        blocks,ovl,ierr,indx;
368:   PetscTruth flg;
369:   const char *type[] = {"none","restrict","interpolate","basic"};

372:   PetscOptionsHead("Additive Schwarz options");
373:     PetscOptionsInt("-pc_asm_blocks","Number of subdomains","PCASMSetTotalSubdomains",osm->n,&blocks,&flg);
374:     if (flg) {PCASMSetTotalSubdomains(pc,blocks,PETSC_NULL); }
375:     PetscOptionsInt("-pc_asm_overlap","Number of grid points overlap","PCASMSetOverlap",osm->overlap,&ovl,&flg);
376:     if (flg) {PCASMSetOverlap(pc,ovl); }
377:     PetscOptionsName("-pc_asm_in_place","Perform matrix factorization inplace","PCASMSetUseInPlace",&flg);
378:     if (flg) {PCASMSetUseInPlace(pc); }
379:     PetscOptionsEList("-pc_asm_type","Type of restriction/extension","PCASMSetType",type,4,type[1],&indx,&flg);
380:     if (flg) {
381:       PCASMSetType(pc,(PCASMType)indx);
382:     }
383:   PetscOptionsTail();
384:   return(0);
385: }

387: /*------------------------------------------------------------------------------------*/

389: EXTERN_C_BEGIN
392: int PCASMSetLocalSubdomains_ASM(PC pc,int n,IS is[])
393: {
394:   PC_ASM *osm = (PC_ASM*)pc->data;

397:   if (n < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Each process must have 0 or more blocks");

399:   if (pc->setupcalled && n != osm->n_local_true) {
400:     SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"PCASMSetLocalSubdomains() should be called before calling PCSetup().");
401:   }
402:   if (!pc->setupcalled){
403:     osm->n_local_true = n;
404:     osm->is           = is;
405:   }
406:   return(0);
407: }
408: EXTERN_C_END

410: EXTERN_C_BEGIN
413: int PCASMSetTotalSubdomains_ASM(PC pc,int N,IS *is)
414: {
415:   PC_ASM *osm = (PC_ASM*)pc->data;
416:   int    rank,size,ierr,n;


420:   if (is) SETERRQ(PETSC_ERR_SUP,"Use PCASMSetLocalSubdomains() to set specific index sets\n\
421: they cannot be set globally yet.");

423:   /*
424:      Split the subdomains equally amoung all processors 
425:   */
426:   MPI_Comm_rank(pc->comm,&rank);
427:   MPI_Comm_size(pc->comm,&size);
428:   n = N/size + ((N % size) > rank);
429:   if (pc->setupcalled && n != osm->n_local_true) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"PCASMSetTotalSubdomains() should be called before PCSetup().");
430:   if (!pc->setupcalled){
431:     osm->n_local_true = n;
432:     osm->is           = 0;
433:   }
434:   return(0);
435: }
436: EXTERN_C_END

438: EXTERN_C_BEGIN
441: int PCASMSetOverlap_ASM(PC pc,int ovl)
442: {
443:   PC_ASM *osm;

446:   if (ovl < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Negative overlap value requested");

448:   osm               = (PC_ASM*)pc->data;
449:   osm->overlap      = ovl;
450:   return(0);
451: }
452: EXTERN_C_END

454: EXTERN_C_BEGIN
457: int PCASMSetType_ASM(PC pc,PCASMType type)
458: {
459:   PC_ASM *osm;

462:   osm        = (PC_ASM*)pc->data;
463:   osm->type  = type;
464:   return(0);
465: }
466: EXTERN_C_END

468: EXTERN_C_BEGIN
471: int PCASMGetSubKSP_ASM(PC pc,int *n_local,int *first_local,KSP **ksp)
472: {
473:   PC_ASM   *jac = (PC_ASM*)pc->data;
474:   int      ierr;

477:   if (jac->n_local_true < 0) {
478:     SETERRQ(1,"Need to call PCSetUP() on PC (or KSPSetUp() on the outer KSP object) before calling here");
479:   }

481:   if (n_local)     *n_local     = jac->n_local_true;
482:   if (first_local) {
483:     MPI_Scan(&jac->n_local_true,first_local,1,MPI_INT,MPI_SUM,pc->comm);
484:     *first_local -= jac->n_local_true;
485:   }
486:   *ksp                         = jac->ksp;
487:   jac->same_local_solves        = PETSC_FALSE; /* Assume that local solves are now different;
488:                                       not necessarily true though!  This flag is 
489:                                       used only for PCView_ASM() */
490:   return(0);
491: }
492: EXTERN_C_END

494: EXTERN_C_BEGIN
497: int PCASMSetUseInPlace_ASM(PC pc)
498: {
499:   PC_ASM *dir;

502:   dir          = (PC_ASM*)pc->data;
503:   dir->inplace = PETSC_TRUE;
504:   return(0);
505: }
506: EXTERN_C_END

508: /*----------------------------------------------------------------------------*/
511: /*@
512:    PCASMSetUseInPlace - Tells the system to destroy the matrix after setup is done.

514:    Collective on PC

516:    Input Parameters:
517: .  pc - the preconditioner context

519:    Options Database Key:
520: .  -pc_asm_in_place - Activates in-place factorization

522:    Note:
523:    PCASMSetUseInplace() can only be used with the KSP method KSPPREONLY, and
524:    when the original matrix is not required during the Solve process.
525:    This destroys the matrix, early thus, saving on memory usage.

527:    Level: intermediate

529: .keywords: PC, set, factorization, direct, inplace, in-place, ASM

531: .seealso: PCILUSetUseInPlace(), PCLUSetUseInPlace ()
532: @*/
533: int PCASMSetUseInPlace(PC pc)
534: {
535:   int ierr,(*f)(PC);

539:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetUseInPlace_C",(void (**)(void))&f);
540:   if (f) {
541:     (*f)(pc);
542:   }
543:   return(0);
544: }
545: /*----------------------------------------------------------------------------*/

549: /*@C
550:     PCASMSetLocalSubdomains - Sets the local subdomains (for this processor
551:     only) for the additive Schwarz preconditioner. 

553:     Collective on PC 

555:     Input Parameters:
556: +   pc - the preconditioner context
557: .   n - the number of subdomains for this processor (default value = 1)
558: -   is - the index sets that define the subdomains for this processor
559:          (or PETSC_NULL for PETSc to determine subdomains)

561:     Notes:
562:     The IS numbering is in the parallel, global numbering of the vector.

564:     By default the ASM preconditioner uses 1 block per processor.  

566:     These index sets cannot be destroyed until after completion of the
567:     linear solves for which the ASM preconditioner is being used.

569:     Use PCASMSetTotalSubdomains() to set the subdomains for all processors.

571:     Level: advanced

573: .keywords: PC, ASM, set, local, subdomains, additive Schwarz

575: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
576:           PCASMCreateSubdomains2D(), PCASMGetLocalSubdomains()
577: @*/
578: int PCASMSetLocalSubdomains(PC pc,int n,IS is[])
579: {
580:   int ierr,(*f)(PC,int,IS[]);

584:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetLocalSubdomains_C",(void (**)(void))&f);
585:   if (f) {
586:     (*f)(pc,n,is);
587:   }
588:   return(0);
589: }

593: /*@C
594:     PCASMSetTotalSubdomains - Sets the subdomains for all processor for the 
595:     additive Schwarz preconditioner.  Either all or no processors in the
596:     PC communicator must call this routine, with the same index sets.

598:     Collective on PC

600:     Input Parameters:
601: +   pc - the preconditioner context
602: .   n - the number of subdomains for all processors
603: -   is - the index sets that define the subdomains for all processor
604:          (or PETSC_NULL for PETSc to determine subdomains)

606:     Options Database Key:
607:     To set the total number of subdomain blocks rather than specify the
608:     index sets, use the option
609: .    -pc_asm_blocks <blks> - Sets total blocks

611:     Notes:
612:     Currently you cannot use this to set the actual subdomains with the argument is.

614:     By default the ASM preconditioner uses 1 block per processor.  

616:     These index sets cannot be destroyed until after completion of the
617:     linear solves for which the ASM preconditioner is being used.

619:     Use PCASMSetLocalSubdomains() to set local subdomains.

621:     Level: advanced

623: .keywords: PC, ASM, set, total, global, subdomains, additive Schwarz

625: .seealso: PCASMSetLocalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
626:           PCASMCreateSubdomains2D()
627: @*/
628: int PCASMSetTotalSubdomains(PC pc,int N,IS *is)
629: {
630:   int ierr,(*f)(PC,int,IS *);

634:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetTotalSubdomains_C",(void (**)(void))&f);
635:   if (f) {
636:     (*f)(pc,N,is);
637:   }
638:   return(0);
639: }

643: /*@
644:     PCASMSetOverlap - Sets the overlap between a pair of subdomains for the
645:     additive Schwarz preconditioner.  Either all or no processors in the
646:     PC communicator must call this routine. 

648:     Collective on PC

650:     Input Parameters:
651: +   pc  - the preconditioner context
652: -   ovl - the amount of overlap between subdomains (ovl >= 0, default value = 1)

654:     Options Database Key:
655: .   -pc_asm_overlap <ovl> - Sets overlap

657:     Notes:
658:     By default the ASM preconditioner uses 1 block per processor.  To use
659:     multiple blocks per perocessor, see PCASMSetTotalSubdomains() and
660:     PCASMSetLocalSubdomains() (and the option -pc_asm_blocks <blks>).

662:     The overlap defaults to 1, so if one desires that no additional
663:     overlap be computed beyond what may have been set with a call to
664:     PCASMSetTotalSubdomains() or PCASMSetLocalSubdomains(), then ovl
665:     must be set to be 0.  In particular, if one does not explicitly set
666:     the subdomains an application code, then all overlap would be computed
667:     internally by PETSc, and using an overlap of 0 would result in an ASM 
668:     variant that is equivalent to the block Jacobi preconditioner.  

670:     Note that one can define initial index sets with any overlap via
671:     PCASMSetTotalSubdomains() or PCASMSetLocalSubdomains(); the routine
672:     PCASMSetOverlap() merely allows PETSc to extend that overlap further
673:     if desired.

675:     Level: intermediate

677: .keywords: PC, ASM, set, overlap

679: .seealso: PCASMSetTotalSubdomains(), PCASMSetLocalSubdomains(), PCASMGetSubKSP(),
680:           PCASMCreateSubdomains2D(), PCASMGetLocalSubdomains()
681: @*/
682: int PCASMSetOverlap(PC pc,int ovl)
683: {
684:   int ierr,(*f)(PC,int);

688:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetOverlap_C",(void (**)(void))&f);
689:   if (f) {
690:     (*f)(pc,ovl);
691:   }
692:   return(0);
693: }

697: /*@
698:     PCASMSetType - Sets the type of restriction and interpolation used
699:     for local problems in the additive Schwarz method.

701:     Collective on PC

703:     Input Parameters:
704: +   pc  - the preconditioner context
705: -   type - variant of ASM, one of
706: .vb
707:       PC_ASM_BASIC       - full interpolation and restriction
708:       PC_ASM_RESTRICT    - full restriction, local processor interpolation
709:       PC_ASM_INTERPOLATE - full interpolation, local processor restriction
710:       PC_ASM_NONE        - local processor restriction and interpolation
711: .ve

713:     Options Database Key:
714: .   -pc_asm_type [basic,restrict,interpolate,none] - Sets ASM type

716:     Level: intermediate

718: .keywords: PC, ASM, set, type

720: .seealso: PCASMSetTotalSubdomains(), PCASMSetTotalSubdomains(), PCASMGetSubKSP(),
721:           PCASMCreateSubdomains2D()
722: @*/
723: int PCASMSetType(PC pc,PCASMType type)
724: {
725:   int ierr,(*f)(PC,PCASMType);

729:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetType_C",(void (**)(void))&f);
730:   if (f) {
731:     (*f)(pc,type);
732:   }
733:   return(0);
734: }

738: /*@C
739:    PCASMGetSubKSP - Gets the local KSP contexts for all blocks on
740:    this processor.
741:    
742:    Collective on PC iff first_local is requested

744:    Input Parameter:
745: .  pc - the preconditioner context

747:    Output Parameters:
748: +  n_local - the number of blocks on this processor or PETSC_NULL
749: .  first_local - the global number of the first block on this processor or PETSC_NULL,
750:                  all processors must request or all must pass PETSC_NULL
751: -  ksp - the array of KSP contexts

753:    Note:  
754:    After PCASMGetSubKSP() the array of KSPes is not to be freed

756:    Currently for some matrix implementations only 1 block per processor 
757:    is supported.
758:    
759:    You must call KSPSetUp() before calling PCASMGetSubKSP().

761:    Level: advanced

763: .keywords: PC, ASM, additive Schwarz, get, sub, KSP, context

765: .seealso: PCASMSetTotalSubdomains(), PCASMSetTotalSubdomains(), PCASMSetOverlap(),
766:           PCASMCreateSubdomains2D(),
767: @*/
768: int PCASMGetSubKSP(PC pc,int *n_local,int *first_local,KSP *ksp[])
769: {
770:   int ierr,(*f)(PC,int*,int*,KSP **);

775:   PetscObjectQueryFunction((PetscObject)pc,"PCASMGetSubKSP_C",(void (**)(void))&f);
776:   if (f) {
777:     (*f)(pc,n_local,first_local,ksp);
778:   } else {
779:     SETERRQ(1,"Cannot get subksp for this type of PC");
780:   }

782:  return(0);
783: }

785: /* -------------------------------------------------------------------------------------*/
786: /*MC
787:    PCASM - Use the (restricted) additive Schwarz method, each block is (approximately) solved with 
788:            its own KSP object.

790:    Options Database Keys:
791: +  -pc_asm_truelocal - Activates PCASMSetUseTrueLocal()
792: .  -pc_asm_in_place - Activates in-place factorization
793: .  -pc_asm_blocks <blks> - Sets total blocks
794: .  -pc_asm_overlap <ovl> - Sets overlap
795: -  -pc_asm_type [basic,restrict,interpolate,none] - Sets ASM type

797:      IMPORTANT: If you run with, for example, 3 blocks on 1 processor or 3 blocks on 3 processors you 
798:       will get a different convergence rate due to the default option of -pc_asm_type restrict. Use
799:       -pc_asm_type basic to use the standard ASM. 

801:    Notes: Each processor can have one or more blocks, but a block cannot be shared by more
802:      than one processor. Defaults to one block per processor.

804:      To set options on the solvers for each block append -sub_ to all the KSP, and PC
805:         options database keys. For example, -sub_pc_type ilu -sub_pc_ilu_levels 1 -sub_ksp_type preonly
806:         
807:      To set the options on the solvers seperate for each block call PCASMGetSubKSP()
808:          and set the options directly on the resulting KSP object (you can access its PC
809:          with KSPGetPC())


812:    Level: beginner

814:    Concepts: additive Schwarz method

816: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
817:            PCBJACOBI, PCASMSetUseTrueLocal(), PCASMGetSubKSP(), PCASMSetLocalSubdomains(),
818:            PCASMSetTotalSubdomains(), PCSetModifySubmatrices(), PCASMSetOverlap(), PCASMSetType(),
819:            PCASMSetUseInPlace()
820: M*/

822: EXTERN_C_BEGIN
825: int PCCreate_ASM(PC pc)
826: {
827:   int    ierr;
828:   PC_ASM *osm;

831:   PetscNew(PC_ASM,&osm);
832:   PetscLogObjectMemory(pc,sizeof(PC_ASM));
833:   PetscMemzero(osm,sizeof(PC_ASM));
834:   osm->n                 = PETSC_DECIDE;
835:   osm->n_local           = 0;
836:   osm->n_local_true      = PETSC_DECIDE;
837:   osm->overlap           = 1;
838:   osm->is_flg            = PETSC_FALSE;
839:   osm->ksp              = 0;
840:   osm->scat              = 0;
841:   osm->is                = 0;
842:   osm->mat               = 0;
843:   osm->pmat              = 0;
844:   osm->type              = PC_ASM_RESTRICT;
845:   osm->same_local_solves = PETSC_TRUE;
846:   osm->inplace           = PETSC_FALSE;
847:   pc->data               = (void*)osm;

849:   pc->ops->apply             = PCApply_ASM;
850:   pc->ops->applytranspose    = PCApplyTranspose_ASM;
851:   pc->ops->setup             = PCSetUp_ASM;
852:   pc->ops->destroy           = PCDestroy_ASM;
853:   pc->ops->setfromoptions    = PCSetFromOptions_ASM;
854:   pc->ops->setuponblocks     = PCSetUpOnBlocks_ASM;
855:   pc->ops->view              = PCView_ASM;
856:   pc->ops->applyrichardson   = 0;

858:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetLocalSubdomains_C","PCASMSetLocalSubdomains_ASM",
859:                     PCASMSetLocalSubdomains_ASM);
860:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetTotalSubdomains_C","PCASMSetTotalSubdomains_ASM",
861:                     PCASMSetTotalSubdomains_ASM);
862:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetOverlap_C","PCASMSetOverlap_ASM",
863:                     PCASMSetOverlap_ASM);
864:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetType_C","PCASMSetType_ASM",
865:                     PCASMSetType_ASM);
866:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMGetSubKSP_C","PCASMGetSubKSP_ASM",
867:                     PCASMGetSubKSP_ASM);
868: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetUseInPlace_C","PCASMSetUseInPlace_ASM",
869:                     PCASMSetUseInPlace_ASM);
870:   return(0);
871: }
872: EXTERN_C_END


877: /*@
878:    PCASMCreateSubdomains2D - Creates the index sets for the overlapping Schwarz 
879:    preconditioner for a two-dimensional problem on a regular grid.

881:    Not Collective

883:    Input Parameters:
884: +  m, n - the number of mesh points in the x and y directions
885: .  M, N - the number of subdomains in the x and y directions
886: .  dof - degrees of freedom per node
887: -  overlap - overlap in mesh lines

889:    Output Parameters:
890: +  Nsub - the number of subdomains created
891: -  is - the array of index sets defining the subdomains

893:    Note:
894:    Presently PCAMSCreateSubdomains2d() is valid only for sequential
895:    preconditioners.  More general related routines are
896:    PCASMSetTotalSubdomains() and PCASMSetLocalSubdomains().

898:    Level: advanced

900: .keywords: PC, ASM, additive Schwarz, create, subdomains, 2D, regular grid

902: .seealso: PCASMSetTotalSubdomains(), PCASMSetLocalSubdomains(), PCASMGetSubKSP(),
903:           PCASMSetOverlap()
904: @*/
905: int PCASMCreateSubdomains2D(int m,int n,int M,int N,int dof,int overlap,int *Nsub,IS **is)
906: {
907:   int i,j,height,width,ystart,xstart,yleft,yright,xleft,xright,loc_outter;
908:   int nidx,*idx,loc,ii,jj,ierr,count;

911:   if (dof != 1) SETERRQ(PETSC_ERR_SUP," ");

913:   *Nsub = N*M;
914:   PetscMalloc((*Nsub)*sizeof(IS **),is);
915:   ystart = 0;
916:   loc_outter = 0;
917:   for (i=0; i<N; i++) {
918:     height = n/N + ((n % N) > i); /* height of subdomain */
919:     if (height < 2) SETERRQ(1,"Too many N subdomains for mesh dimension n");
920:     yleft  = ystart - overlap; if (yleft < 0) yleft = 0;
921:     yright = ystart + height + overlap; if (yright > n) yright = n;
922:     xstart = 0;
923:     for (j=0; j<M; j++) {
924:       width = m/M + ((m % M) > j); /* width of subdomain */
925:       if (width < 2) SETERRQ(1,"Too many M subdomains for mesh dimension m");
926:       xleft  = xstart - overlap; if (xleft < 0) xleft = 0;
927:       xright = xstart + width + overlap; if (xright > m) xright = m;
928:       nidx   = (xright - xleft)*(yright - yleft);
929:       PetscMalloc(nidx*sizeof(int),&idx);
930:       loc    = 0;
931:       for (ii=yleft; ii<yright; ii++) {
932:         count = m*ii + xleft;
933:         for (jj=xleft; jj<xright; jj++) {
934:           idx[loc++] = count++;
935:         }
936:       }
937:       ISCreateGeneral(PETSC_COMM_SELF,nidx,idx,(*is)+loc_outter++);
938:       PetscFree(idx);
939:       xstart += width;
940:     }
941:     ystart += height;
942:   }
943:   for (i=0; i<*Nsub; i++) { ISSort((*is)[i]); }
944:   return(0);
945: }

949: /*@C
950:     PCASMGetLocalSubdomains - Gets the local subdomains (for this processor
951:     only) for the additive Schwarz preconditioner. 

953:     Collective on PC 

955:     Input Parameter:
956: .   pc - the preconditioner context

958:     Output Parameters:
959: +   n - the number of subdomains for this processor (default value = 1)
960: -   is - the index sets that define the subdomains for this processor
961:          

963:     Notes:
964:     The IS numbering is in the parallel, global numbering of the vector.

966:     Level: advanced

968: .keywords: PC, ASM, set, local, subdomains, additive Schwarz

970: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
971:           PCASMCreateSubdomains2D(), PCASMSetLocalSubdomains(), PCASMGetLocalSubmatrices()
972: @*/
973: int PCASMGetLocalSubdomains(PC pc,int *n,IS *is[])
974: {
975:   PC_ASM *osm;

980:   if (!pc->setupcalled) {
981:     SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call after KSPSetUP() or PCSetUp().");
982:   }

984:   osm = (PC_ASM*)pc->data;
985:   if (n)  *n = osm->n_local_true;
986:   if (is) *is = osm->is;
987:   return(0);
988: }

992: /*@C
993:     PCASMGetLocalSubmatrices - Gets the local submatrices (for this processor
994:     only) for the additive Schwarz preconditioner. 

996:     Collective on PC 

998:     Input Parameter:
999: .   pc - the preconditioner context

1001:     Output Parameters:
1002: +   n - the number of matrices for this processor (default value = 1)
1003: -   mat - the matrices
1004:          

1006:     Level: advanced

1008: .keywords: PC, ASM, set, local, subdomains, additive Schwarz, block Jacobi

1010: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
1011:           PCASMCreateSubdomains2D(), PCASMSetLocalSubdomains(), PCASMGetLocalSubdomains()
1012: @*/
1013: int PCASMGetLocalSubmatrices(PC pc,int *n,Mat *mat[])
1014: {
1015:   PC_ASM *osm;

1020:   if (!pc->setupcalled) {
1021:     SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call after KSPSetUP() or PCSetUp().");
1022:   }

1024:   osm = (PC_ASM*)pc->data;
1025:   if (n)   *n   = osm->n_local_true;
1026:   if (mat) *mat = osm->pmat;
1027:   return(0);
1028: }